Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Restore native TrueNAS disk temperature History in the drawer
pulse-triage[bot]4c57c21bcc2e
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex 0722e0be84..26197f3180 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -2349,6 +2349,40 @@ truthfulness, not native thaw, containing-release or workload acceptance.## Current State+### TrueNAS physical-disk History — issue #2519++The existing disk drawer's `smart_temp` chart consumes the same tenant-scoped+native temperature-history provider as physical-disk sparklines when local+history is shallow. Sufficient local coverage and unrelated SMART/I/O metrics+make no new native call. Native Celsius values retain their original sample+times; downsampling and existing canonical metrics-target resolution remain.+Neither chart pads a missing or single sample into a fabricated flat history.++A successful `reporting.get_data` response with missing disk series permits+one `reporting.netdata_get_data` query for those identifiers alone. Both reads+share the original start/end window, `aggregate:true`, authenticated session+and operation deadline. Failure of the first method does not select another+method, transport or credential. Failure of the second is retained alongside+independently successful series; missing disks do not become observed zero.+Only requested, valid timestamped samples inside the requested window are+admitted, never aggregation summaries, future values or unrequested identities.++Providers map native disk names through the current inventory to canonical+metric IDs, detach returned series and reject ambiguous aliases. In-flight+inventory replacement cannot attach an old disk's series to a new disk at its+former device name. Pollers isolate tenants, reject cross-connection ambiguous+metric IDs and recheck provider ownership after reads so removed/replaced+connections cannot serve stale in-flight output. No host-temperature summary,+CPU substitution, permission widening, store schema or retention change is+introduced. Native appliance and containing-release acceptance remain separate+from synthetic client/provider/authenticated-route controls.++`disk_history_netdata_test.go`, `disk_history_identity_test.go`,+`truenas_disk_history_scope_test.go`, `disk_drawer_native_history_test.go` and+`truenas_disk_history_integration_test.go` cover method distinction, partial+failure, empty/invalid/windowed samples, canonical identity, revocation,+existing reader behaviour and the real authenticated/scoped Pulse route.+V2 partial inventories preserve usable filesystem readings while leaving typed-helper health degraded. Global inventory failure still yields no inventory. No host-network exception, guest mutation, database/retention change or temperature substitution accompanies this repair.### Discovery observes VM guest execution pausesdiff --git a/docs/release-control/v6/internal/subsystems/registry.json b/docs/release-control/v6/internal/subsystems/registry.jsonindex d79af8c40e..cd00c396ca 100644--- a/docs/release-control/v6/internal/subsystems/registry.json+++ b/docs/release-control/v6/internal/subsystems/registry.json@@ -6156,7 +6156,8 @@"internal/monitoring/monitor_pbs_coverage_test.go","internal/monitoring/monitor_pmg_test.go","internal/monitoring/monitor_polling_test.go",- "internal/monitoring/truenas_poller_test.go"+ "internal/monitoring/truenas_poller_test.go",+ "internal/monitoring/truenas_disk_history_scope_test.go"]},{@@ -6212,7 +6213,10 @@"internal/truenas/core_reporting_test.go","internal/truenas/provider_pool_health_contract_test.go","internal/truenas/provider_test.go",- "internal/truenas/transport_test.go"+ "internal/truenas/transport_test.go",+ "internal/truenas/disk_history_identity_test.go",+ "internal/truenas/disk_history_netdata_test.go",+ "internal/api/truenas_disk_history_integration_test.go"]},{@@ -6226,7 +6230,9 @@"test_prefixes": [],"exact_files": ["internal/monitoring/monitor_metrics_chart_batch_bench_test.go",- "internal/monitoring/monitor_metrics_slo_test.go"+ "internal/monitoring/monitor_metrics_slo_test.go",+ "internal/monitoring/disk_drawer_native_history_test.go",+ "internal/api/truenas_disk_history_integration_test.go"]},{diff --git a/internal/api/truenas_disk_history_integration_test.go b/internal/api/truenas_disk_history_integration_test.gonew file mode 100644index 0000000000..00d75fff24--- /dev/null+++ b/internal/api/truenas_disk_history_integration_test.go@@ -0,0 +1,203 @@+package api++import (+ "context"+ "encoding/json"+ "net/http"+ "net/http/httptest"+ "os"+ "path/filepath"+ "sync/atomic"+ "testing"+ "time"++ "github.com/gorilla/websocket"+ "github.com/rcourtman/pulse-go-rewrite/internal/config"+ "github.com/rcourtman/pulse-go-rewrite/internal/models"+ "github.com/rcourtman/pulse-go-rewrite/internal/monitoring"+ "github.com/rcourtman/pulse-go-rewrite/internal/truenas"+ "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"+)++// Inventory is a declared fixture; History traverses the real authenticated+// TrueNAS client, provider, tenant poller, monitor and Pulse route. It does not+// claim an appliance installation, rendered browser or reporter retest.+type nativeDiskHistoryAPIFetcher struct {+ truenas.FixtureFetcher+ truenas.APIFetcher+}++func (f *nativeDiskHistoryAPIFetcher) Fetch(ctx context.Context) (*truenas.FixtureSnapshot, error) {+ return f.FixtureFetcher.Fetch(ctx)+}++func TestTrueNASDiskHistoryAuthenticatedRoute(t *testing.T) {+ previous := truenas.IsFeatureEnabled()+ truenas.SetFeatureEnabled(true)+ t.Cleanup(func() { truenas.SetFeatureEnabled(previous) })+ previousMulti := IsMultiTenantEnabled()+ SetMultiTenantEnabled(true)+ t.Cleanup(func() { SetMultiTenantEnabled(previousMulti) })+ t.Setenv("PULSE_DEV", "true")+ var reads, auths atomic.Int32+ var nativeEnd atomic.Int64+ upgrader := websocket.Upgrader{}+ native := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.URL.Path != "/api/current" {+ t.Errorf("unexpected REST/transport fallback %s", r.URL.Path)+ http.NotFound(w, r)+ return+ }+ conn, err := upgrader.Upgrade(w, r, nil)+ if err != nil {+ t.Error(err)+ return+ }+ defer conn.Close()+ for {+ var request struct {+ ID any `json:"id"`+ Method string `json:"method"`+ Params []json.RawMessage `json:"params"`+ }+ if err := conn.ReadJSON(&request); err != nil {+ return+ }+ var result any+ switch request.Method {+ case "auth.login_with_api_key":+ var key string+ if len(request.Params) != 1 || json.Unmarshal(request.Params[0], &key) != nil || key != "native-fixture-key" {+ t.Error("native authentication contract changed")+ return+ }+ auths.Add(1)+ result = true+ case "reporting.get_data", "reporting.netdata_get_data":+ reads.Add(1)+ if len(request.Params) != 2 {+ t.Error("native history parameters changed")+ return+ }+ var graphs []struct {+ Name string `json:"name"`+ Identifier string `json:"identifier"`+ }+ var query struct {+ Start int64 `json:"start"`+ End int64 `json:"end"`+ Aggregate bool `json:"aggregate"`+ }+ if json.Unmarshal(request.Params[0], &graphs) != nil || len(graphs) != 1 || graphs[0].Name != "disktemp" || graphs[0].Identifier != "sdb" || json.Unmarshal(request.Params[1], &query) != nil || !query.Aggregate || query.End-query.Start != 3600 {+ t.Error("native disk identity/time window changed")+ return+ }+ result = []any{}+ if request.Method == "reporting.netdata_get_data" {+ nativeEnd.Store(query.End)+ result = []any{map[string]any{+ "name": "disktemp", "identifier": "sdb", "legend": []string{"temperature"},+ "start": query.End - 3540, "end": query.End - 60, "step": 1740,+ "data": [][]any{{31.25}, {33.5}, {32.75}},+ "aggregations": map[string]any{"min": []float64{999}, "mean": []float64{999}, "max": []float64{999}},+ }}+ }+ default:+ t.Errorf("unexpected native method %q", request.Method)+ return+ }+ if err := conn.WriteJSON(map[string]any{"jsonrpc": "2.0", "id": request.ID, "result": result}); err != nil {+ t.Error(err)+ return+ }+ }+ }))+ t.Cleanup(native.Close)+ nativeClient, err := truenas.NewClient(truenas.ClientConfig{Host: native.URL, APIKey: "native-fixture-key", UseHTTPS: true, InsecureSkipVerify: true})+ if err != nil {+ t.Fatal(err)+ }+ t.Cleanup(nativeClient.Close)+ fixture := truenas.DefaultFixtures()+ fixture.Disks = []truenas.Disk{{Name: "sdb", Serial: "disk-serial", Temperature: 34}}+ provider := truenas.NewLiveProviderForConnection(&nativeDiskHistoryAPIFetcher{FixtureFetcher: truenas.FixtureFetcher{Snapshot: fixture}, APIFetcher: truenas.APIFetcher{Client: nativeClient}}, "native-connection")+ if err := provider.Refresh(context.Background()); err != nil {+ t.Fatal(err)+ }+ poller := monitoring.NewTrueNASPoller(nil, time.Minute, nil)+ setUnexportedField(t, poller, "providersByOrg", map[string]map[string]*truenas.Provider{"org-a": {"native-connection": provider}})+ setUnexportedField(t, poller, "cachedRecordsByOrg", map[string]map[string][]unifiedresources.IngestRecord{"org-a": {"native-connection": provider.Records()}})+ monitor := &monitoring.Monitor{}+ monitor.SetOrgID("org-a")+ setUnexportedField(t, monitor, "state", models.NewState())+ setUnexportedField(t, monitor, "metricsHistory", monitoring.NewMetricsHistory(20, time.Hour))+ monitor.SetSupplementalRecordsProvider(unifiedresources.SourceTrueNAS, poller)+ registry := unifiedresources.NewRegistry(nil)+ monitor.SetResourceStore(unifiedresources.NewMonitorAdapter(registry))++ const token = "disk-history-read-fixture.12345678"+ readToken := newTokenRecord(t, token, []string{config.ScopeMonitoringRead}, nil)+ readToken.OrgID = "org-a"+ const wrongScope = "disk-history-settings-fixture.12345678"+ scopeToken := newTokenRecord(t, wrongScope, []string{config.ScopeSettingsRead}, nil)+ scopeToken.OrgID = "org-a"+ cfg := newTestConfigWithTokens(t, readToken, scopeToken)+ cfg.AuthUser = "fixture-admin"+ cfg.AuthPass = "fixture-password-hash-not-used-by-token"+ for _, org := range []string{"org-a", "org-b"} {+ if err := os.MkdirAll(filepath.Join(cfg.DataPath, "orgs", org), 0700); err != nil {+ t.Fatal(err)+ }+ }+ router := NewRouter(cfg, monitor, nil, nil, nil, "test")+ t.Cleanup(router.ShutdownBackgroundWorkers)+ t.Cleanup(router.ShutdownResourceStores)+ t.Cleanup(router.ShutdownRBAC)+ mtm := monitoring.NewMultiTenantMonitor(cfg, nil, nil)+ setUnexportedField(t, mtm, "monitors", map[string]*monitoring.Monitor{"org-a": monitor, "org-b": {}})+ router.mtMonitor = mtm+ request := func(raw, org string) *httptest.ResponseRecorder {+ req := httptest.NewRequest(http.MethodGet, "/api/metrics-store/history?resourceType=disk&resourceId=disk-serial&metric=smart_temp&range=1h", nil)+ req.Header.Set("X-Pulse-Org-ID", org)+ if raw != "" {+ req.Header.Set("X-API-Token", raw)+ }+ rec := httptest.NewRecorder()+ router.Handler().ServeHTTP(rec, req)+ return rec+ }+ for _, test := range []struct {+ raw, org string+ want int+ }{{"", "org-a", http.StatusUnauthorized}, {wrongScope, "org-a", http.StatusForbidden}, {token, "org-b", http.StatusForbidden}} {+ rec := request(test.raw, test.org)+ if rec.Code != test.want || reads.Load() != 0 {+ t.Fatalf("auth/scope/tenant refusal lost: status %d want %d native reads %d body %s", rec.Code, test.want, reads.Load(), rec.Body.String())+ }+ }+ rec := request(token, "org-a")+ if rec.Code != http.StatusOK {+ t.Fatalf("authenticated History status %d: %s", rec.Code, rec.Body.String())+ }+ var got metricsHistoryResponse+ if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil {+ t.Fatal(err)+ }+ if got.ResourceType != "disk" || got.ResourceId != "disk-serial" || got.Metric != "smart_temp" || len(got.Points) != 3 || reads.Load() != 2 || auths.Load() != 1 {+ t.Fatalf("client/provider/poller/route did not deliver native disk history: %+v, reads %d auths %d", got, reads.Load(), auths.Load())+ }+ end := nativeEnd.Load()+ for i, value := range []float64{31.25, 33.5, 32.75} {+ if got.Points[i].Timestamp != time.Unix(end-3540+int64(i)*1740, 0).UnixMilli() || got.Points[i].Value != value {+ t.Fatalf("native timestamp/Celsius sample %d changed: %+v", i, got.Points[i])+ }+ }+ if removed := cfg.RemoveAPIToken(readToken.ID); removed == nil {+ t.Fatal("fixture token missing")+ }+ cfg.SortAPITokens()+ rec = request(token, "org-a")+ if rec.Code != http.StatusUnauthorized || reads.Load() != 2 {+ t.Fatalf("revoked token still reached History: status %d reads %d", rec.Code, reads.Load())+ }+}diff --git a/internal/monitoring/disk_drawer_native_history_test.go b/internal/monitoring/disk_drawer_native_history_test.gonew file mode 100644index 0000000000..acbf536b42--- /dev/null+++ b/internal/monitoring/disk_drawer_native_history_test.go@@ -0,0 +1,84 @@+package monitoring++import (+ "reflect"+ "testing"+ "time"++ "github.com/rcourtman/pulse-go-rewrite/internal/mock"+ "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"+)++func TestDiskDrawerNativeTemperatureHistory(t *testing.T) {+ if mock.IsMockEnabled() {+ t.Fatal("native History test requires non-mock mode")+ }+ now := time.Now().UTC().Truncate(time.Second)+ native := []MetricPoint{+ {Timestamp: now.Add(-55 * time.Minute), Value: 31.25},+ {Timestamp: now.Add(-30 * time.Minute), Value: 33.5},+ {Timestamp: now.Add(-time.Minute), Value: 32.75},+ }+ for _, count := range []int{0, 1} {+ t.Run(map[int]string{0: "empty", 1: "sparse"}[count], func(t *testing.T) {+ monitor := &Monitor{metricsHistory: NewMetricsHistory(20, time.Hour)}+ if count != 0 {+ monitor.metricsHistory.AddDiskMetric("disk-serial", "smart_temp", 39, now.Add(-time.Minute))+ }+ monitor.supplementalProviders = map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider{+ unifiedresources.SourceTrueNAS: &stubDiskTemperatureHistoryProvider{history: map[string][]MetricPoint{"disk-serial": native}},+ }+ got := monitor.GetDiskMetricsForChart("disk-serial", "smart_temp", time.Hour)+ if !reflect.DeepEqual(got, native) {+ t.Fatalf("drawer lost native timestamps/Celsius samples: got %+v, want %+v", got, native)+ }+ got[0].Value = 999+ if next := monitor.GetDiskMetricsForChart("disk-serial", "smart_temp", time.Hour); !reflect.DeepEqual(next, native) {+ t.Fatal("drawer response aliases provider history")+ }+ })+ }+}++type countingDiskHistoryProvider struct {+ stubDiskTemperatureHistoryProvider+ calls int+ org string+}++func (p *countingDiskHistoryProvider) PhysicalDiskTemperatureHistory(_ *Monitor, org string, _ time.Duration) map[string][]MetricPoint {+ p.calls+++ p.org = org+ return p.history+}++func TestDiskDrawerNativeHistoryBoundaries(t *testing.T) {+ now := time.Now().UTC().Truncate(time.Second)+ provider := &countingDiskHistoryProvider{stubDiskTemperatureHistoryProvider: stubDiskTemperatureHistoryProvider{+ history: map[string][]MetricPoint{"known-disk": {{Timestamp: now.Add(-time.Minute), Value: 31}}},+ }}+ monitor := &Monitor{metricsHistory: NewMetricsHistory(20, time.Hour)}+ monitor.SetOrgID("org-a")+ monitor.supplementalProviders = map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider{unifiedresources.SourceTrueNAS: provider}+ if got := monitor.GetDiskMetricsForChart("known-disk", "diskread", time.Hour); len(got) != 0 || provider.calls != 0 {+ t.Fatalf("disk I/O must not ask for temperatures: %+v, calls %d", got, provider.calls)+ }+ if got := monitor.GetDiskMetricsForChart("foreign-disk", "smart_temp", time.Hour); len(got) != 0 || provider.org != "org-a" {+ t.Fatalf("foreign/native identity leaked: %+v, org %q", got, provider.org)+ }+ got := monitor.GetDiskMetricsForChart("known-disk", "smart_temp", time.Hour)+ if len(got) != 1 || got[0].Value != 31 || !got[0].Timestamp.Equal(now.Add(-time.Minute)) {+ t.Fatalf("one actual sample was fabricated/renewed: %+v", got)+ }+ calls := provider.calls+ for _, offset := range []time.Duration{-55 * time.Minute, -30 * time.Minute, -time.Minute} {+ monitor.metricsHistory.AddDiskMetric("known-disk", "smart_temp", 35, now.Add(offset))+ }+ if got := monitor.GetDiskMetricsForChart("known-disk", "smart_temp", time.Hour); len(got) != 3 || got[0].Value != 35 || provider.calls != calls {+ t.Fatalf("sufficient local coverage should not invoke native RPC: %+v, calls %d", got, provider.calls)+ }+ var nilMonitor *Monitor+ if got := nilMonitor.GetDiskMetricsForChart("known-disk", "smart_temp", time.Hour); len(got) != 0 {+ t.Fatalf("nil monitor has history: %+v", got)+ }+}diff --git a/internal/monitoring/monitor_metrics.go b/internal/monitoring/monitor_metrics.goindex 0e8d37a4d8..78fef51c5e 100644--- a/internal/monitoring/monitor_metrics.go+++ b/internal/monitoring/monitor_metrics.go@@ -190,7 +190,7 @@ func (m *Monitor) GetDiskMetrics(resourceID string, metricType string, duration// GetDiskMetricsForChart returns physical-disk metrics optimized for chart// display, preferring in-memory data for freshness and falling back to the-// persistent store when coverage is shallow.+// persistent store and source-native temperature history when coverage is shallow.func (m *Monitor) GetDiskMetricsForChart(resourceID string, metricType string, duration time.Duration) []MetricPoint {inMemoryPoints := m.GetDiskMetrics(resourceID, metricType, duration)if mock.IsMockEnabled() {@@ -207,6 +207,14 @@ func (m *Monitor) GetDiskMetricsForChart(resourceID string, metricType string, dbest = cloneMetricSeries(candidate)}}+ if metricType == "smart_temp" && !hasSufficientChartSeriesCoverage(best, duration) {+ if nativePoints := m.nativePhysicalDiskTemperatureHistory(duration)[resourceID]; len(nativePoints) > 0 {+ nativePoints = lttb(nativePoints, chartDownsampleTarget)+ if shouldPreferMetricSeries(best, nativePoints, duration) {+ best = cloneMetricSeries(nativePoints)+ }+ }+ }return best}@@ -267,11 +275,10 @@ func (m *Monitor) GetPhysicalDiskTemperatureCharts(duration time.Duration) map[s// Phase 1: Collect disk metadata and resource IDs.type diskMeta struct {- resourceID string- name string- node string- instance string- temperature int+ resourceID string+ name string+ node string+ instance string}var disks []diskMetafor _, disk := range readState.PhysicalDisks() {@@ -295,11 +302,10 @@ func (m *Monitor) GetPhysicalDiskTemperatureCharts(duration time.Duration) map[sname = strings.TrimSpace(disk.DevPath())}disks = append(disks, diskMeta{- resourceID: resourceID,- name: name,- node: strings.TrimSpace(disk.Node()),- instance: strings.TrimSpace(disk.Instance()),- temperature: disk.Temperature(),+ resourceID: resourceID,+ name: name,+ node: strings.TrimSpace(disk.Node()),+ instance: strings.TrimSpace(disk.Instance()),})}@@ -347,16 +353,8 @@ func (m *Monitor) GetPhysicalDiskTemperatureCharts(duration time.Duration) map[s}}- // Sparklines require >= 2 points. If the store returned 0 or 1 points- // but the disk has a live temperature reading, pad to 2 points so the- // chart can render (flat line at current temperature).- if len(tempPoints) < 2 {- now := time.Now()- tempPoints = []MetricPoint{- {Timestamp: now.Add(-60 * time.Second), Value: float64(d.temperature)},- {Timestamp: now, Value: float64(d.temperature)},- }- }+ // A live temperature is not two historical observations. Leave empty+ // or single-point histories intact instead of inventing a flat line.result[d.resourceID] = DiskChartEntry{Name: d.name,diff --git a/internal/monitoring/monitor_metrics_chart_fallback_test.go b/internal/monitoring/monitor_metrics_chart_fallback_test.goindex df1525dbe2..38540b40e2 100644--- a/internal/monitoring/monitor_metrics_chart_fallback_test.go+++ b/internal/monitoring/monitor_metrics_chart_fallback_test.go@@ -370,13 +370,8 @@ func TestGetPhysicalDiskTemperatureCharts_UsesUnifiedReadStateDiskViews(t *testiif entry.Instance != "lab-a" {t.Fatalf("chart instance = %q, want lab-a", entry.Instance)}- if len(entry.Temperature) != 2 {- t.Fatalf("expected padded 2-point sparkline series, got %d", len(entry.Temperature))- }- for _, point := range entry.Temperature {- if point.Value != 42 {- t.Fatalf("expected canonical disk temperature 42 in padded series, got %.2f", point.Value)- }+ if len(entry.Temperature) != 0 {+ t.Fatalf("a current temperature must not fabricate historical samples: %+v", entry.Temperature)}}diff --git a/internal/monitoring/truenas_disk_history_scope_test.go b/internal/monitoring/truenas_disk_history_scope_test.gonew file mode 100644index 0000000000..f0fe9c8c5a--- /dev/null+++ b/internal/monitoring/truenas_disk_history_scope_test.go@@ -0,0 +1,76 @@+package monitoring++import (+ "context"+ "errors"+ "testing"+ "time"++ "github.com/rcourtman/pulse-go-rewrite/internal/truenas"+)++type scopedDiskHistoryFetcher struct {+ truenas.FixtureFetcher+ points []truenas.TimeSeriesPoint+ err error+ onRead func()+}++func (f *scopedDiskHistoryFetcher) DiskTemperatureHistory(context.Context, []string, time.Duration) (map[string][]truenas.TimeSeriesPoint, error) {+ if f.onRead != nil {+ f.onRead()+ }+ return map[string][]truenas.TimeSeriesPoint{"sdb": f.points}, f.err+}++func TestTrueNASDiskHistoryScopeAndRevocation(t *testing.T) {+ previous := truenas.IsFeatureEnabled()+ truenas.SetFeatureEnabled(true)+ t.Cleanup(func() { truenas.SetFeatureEnabled(previous) })+ points := []truenas.TimeSeriesPoint{{Timestamp: time.Now().UTC().Add(-time.Minute), Value: 33.25}}+ fixtures := truenas.DefaultFixtures()+ fixtures.Disks = []truenas.Disk{{Name: "sdb", Serial: "shared-serial"}}+ fetcherA := &scopedDiskHistoryFetcher{FixtureFetcher: truenas.FixtureFetcher{Snapshot: fixtures}, points: points, err: errors.New("synthetic missing peer series")}+ fetcherB := &scopedDiskHistoryFetcher{FixtureFetcher: truenas.FixtureFetcher{Snapshot: fixtures}, points: []truenas.TimeSeriesPoint{{Timestamp: points[0].Timestamp, Value: 50}}}+ providerA, providerB := truenas.NewLiveProvider(fetcherA), truenas.NewLiveProvider(fetcherB)+ for _, provider := range []*truenas.Provider{providerA, providerB} {+ if err := provider.Refresh(context.Background()); err != nil {+ t.Fatal(err)+ }+ }+ poller := NewTrueNASPoller(nil, time.Minute, nil)+ poller.providersByOrg = map[string]map[string]*truenas.Provider{"org-a": {"conn": providerA}, "org-b": {"conn": providerB}}+ for org, want := range map[string]float64{"org-a": 33.25, "org-b": 50} {+ got := poller.PhysicalDiskTemperatureHistory(nil, org, time.Hour)+ if len(got) != 1 || len(got["shared-serial"]) != 1 || got["shared-serial"][0].Value != want || !got["shared-serial"][0].Timestamp.Equal(points[0].Timestamp) {+ t.Fatalf("tenant/partial/native time lost: org %s %+v", org, got)+ }+ }+ if got := poller.PhysicalDiskTemperatureHistory(nil, "org-missing", time.Hour); len(got) != 0 {+ t.Fatalf("unknown tenant received history: %+v", got)+ }+ poller.providersByOrg["org-a"]["second-conn"] = providerB+ if got := poller.PhysicalDiskTemperatureHistory(nil, "org-a", time.Hour); len(got) != 0 {+ t.Fatalf("ambiguous metric target picked an arbitrary connection: %+v", got)+ }+ delete(poller.providersByOrg["org-a"], "second-conn")+ for _, change := range []string{"remove", "replace"} {+ poller.providersByOrg["org-a"]["conn"] = providerA+ fetcherA.onRead = func() {+ poller.mu.Lock()+ defer poller.mu.Unlock()+ if change == "remove" {+ delete(poller.providersByOrg["org-a"], "conn")+ } else {+ poller.providersByOrg["org-a"]["conn"] = providerB+ }+ }+ if got := poller.PhysicalDiskTemperatureHistory(nil, "org-a", time.Hour); len(got) != 0 {+ t.Fatalf("%s while reading served revoked history: %+v", change, got)+ }+ }+ truenas.SetFeatureEnabled(false)+ if got := poller.PhysicalDiskTemperatureHistory(nil, "org-b", time.Hour); len(got) != 0 {+ t.Fatalf("disabled source received history: %+v", got)+ }+}diff --git a/internal/monitoring/truenas_poller.go b/internal/monitoring/truenas_poller.goindex e8e627fe71..9c5f784936 100644--- a/internal/monitoring/truenas_poller.go+++ b/internal/monitoring/truenas_poller.go@@ -1018,6 +1018,8 @@ func (p *TrueNASPoller) PhysicalDiskTemperatureHistory(_ *Monitor, orgID string,defer cancel()history := make(map[string][]MetricPoint)+ owners := make(map[string]trueNASPollerProviderEntry)+ ambiguous := make(map[string]bool)for _, entry := range entries {nativeHistory, err := entry.provider.PhysicalDiskTemperatureHistory(ctx, duration)if err != nil {@@ -1028,10 +1030,15 @@ func (p *TrueNASPoller) PhysicalDiskTemperatureHistory(_ *Monitor, orgID string,Str("connection_id", strings.TrimSpace(entry.connectionID)).Err(err).Msg("TrueNAS poller failed to read native disk temperature history")- continue}for resourceID, points := range nativeHistory {- if strings.TrimSpace(resourceID) == "" || len(points) == 0 {+ if strings.TrimSpace(resourceID) == "" || len(points) == 0 || ambiguous[resourceID] {+ continue+ }+ if owner, exists := owners[resourceID]; exists && owner.connectionID != entry.connectionID {+ delete(history, resourceID)+ delete(owners, resourceID)+ ambiguous[resourceID] = truecontinue}converted := make([]MetricPoint, len(points))@@ -1042,8 +1049,18 @@ func (p *TrueNASPoller) PhysicalDiskTemperatureHistory(_ *Monitor, orgID string,}}history[resourceID] = converted+ owners[resourceID] = entry}}+ // Do not serve a provider that was removed or replaced while any of the+ // native reads were in flight. No poller lock is held during network I/O.+ p.mu.Lock()+ for resourceID, owner := range owners {+ if !truenas.IsFeatureEnabled() || p.providersByOrg[orgID][owner.connectionID] != owner.provider {+ delete(history, resourceID)+ }+ }+ p.mu.Unlock()if len(history) == 0 {return nil}diff --git a/internal/truenas/client.go b/internal/truenas/client.goindex 75069af77c..a9a1a4fa06 100644--- a/internal/truenas/client.go+++ b/internal/truenas/client.go@@ -3090,7 +3090,53 @@ func (c *trueNASRPCClient) getDiskTemperatureHistory(ctx context.Context, identiif err != nil {return nil, err}- return parseReportingDiskTemperatureHistory(response), nil+ history := boundedDiskTemperatureHistory(response, identifiers, start, end)++ // SCALE can have disk samples in Netdata while reporting.get_data+ // successfully returns no series for those disks (#2519). Fill only those+ // missing series, once, in the same operation budget and time window. An+ // RPC/permission/transport failure above is not an empty-series result and+ // must never cause another method or transport to be tried.+ missingGraphs := make([]map[string]any, 0, len(identifiers))+ missingIdentifiers := make([]string, 0, len(identifiers))+ for _, identifier := range identifiers {+ if len(history[identifier]) == 0 {+ missingGraphs = append(missingGraphs, map[string]any{"name": "disktemp", "identifier": identifier})+ missingIdentifiers = append(missingIdentifiers, identifier)+ }+ }+ if len(missingGraphs) == 0 {+ return history, nil+ }+ response, err = c.getReportingDataForMethod(ctx, "reporting.netdata_get_data", missingGraphs, reportingRangeQuery(start, end))+ if err != nil {+ // Preserve independently successful series with the failure. Consumers+ // may show them, but cannot turn an omitted disk into a healthy zero.+ return history, err+ }+ for identifier, points := range boundedDiskTemperatureHistory(response, missingIdentifiers, start, end) {+ if history == nil {+ history = make(map[string][]TimeSeriesPoint)+ }+ history[identifier] = points+ }+ return history, nil+}++func boundedDiskTemperatureHistory(responses []trueNASReportingGetDataResponse, identifiers []string, start, end int64) map[string][]TimeSeriesPoint {+ parsed := parseReportingDiskTemperatureHistory(responses)+ history := make(map[string][]TimeSeriesPoint)+ for _, identifier := range identifiers {+ for _, point := range parsed[identifier] {+ if timestamp := point.Timestamp.Unix(); timestamp >= start && timestamp <= end {+ history[identifier] = append(history[identifier], point)+ }+ }+ }+ if len(history) == 0 {+ return nil+ }+ return history}func (c *trueNASRPCClient) getReportingData(ctx context.Context, graphs []map[string]any) ([]trueNASReportingGetDataResponse, error) {@@ -3111,6 +3157,10 @@ func (c *trueNASRPCClient) getReportingData(ctx context.Context, graphs []map[st}func (c *trueNASRPCClient) getReportingDataWithQuery(ctx context.Context, graphs []map[string]any, query map[string]any) ([]trueNASReportingGetDataResponse, error) {+ return c.getReportingDataForMethod(ctx, "reporting.get_data", graphs, query)+}++func (c *trueNASRPCClient) getReportingDataForMethod(ctx context.Context, method string, graphs []map[string]any, query map[string]any) ([]trueNASReportingGetDataResponse, error) {if c == nil || c.conn == nil {return nil, fmt.Errorf("truenas rpc connection is nil")}@@ -3126,7 +3176,7 @@ func (c *trueNASRPCClient) getReportingDataWithQuery(ctx context.Context, graphsquery,}var response []trueNASReportingGetDataResponse- if err := c.call(ctx, "reporting.get_data", params, &response); err != nil {+ if err := c.call(ctx, method, params, &response); err != nil {return nil, err}return response, nildiff --git a/internal/truenas/disk_history_identity_test.go b/internal/truenas/disk_history_identity_test.gonew file mode 100644index 0000000000..ea3da9be48--- /dev/null+++ b/internal/truenas/disk_history_identity_test.go@@ -0,0 +1,73 @@+package truenas++import (+ "context"+ "errors"+ "reflect"+ "testing"+ "time"+)++type diskHistoryResultFetcher struct {+ FixtureFetcher+ history map[string][]TimeSeriesPoint+ err error+ beforeReturn func()+ queried []string+}++func (f *diskHistoryResultFetcher) DiskTemperatureHistory(_ context.Context, ids []string, _ time.Duration) (map[string][]TimeSeriesPoint, error) {+ f.queried = append([]string(nil), ids...)+ if f.beforeReturn != nil {+ f.beforeReturn()+ }+ return f.history, f.err+}++func TestDiskHistoryProviderPreservesPartialAndCurrentIdentity(t *testing.T) {+ now := time.Now().UTC()+ points := []TimeSeriesPoint{{Timestamp: now.Add(-time.Minute), Value: 32.25}}+ missing := errors.New("synthetic missing-series error")+ fixtures := DefaultFixtures()+ fixtures.Disks = []Disk{{Name: "sdb", Serial: "CURRENT-SERIAL"}}+ fetcher := &diskHistoryResultFetcher{FixtureFetcher: FixtureFetcher{Snapshot: fixtures}, history: map[string][]TimeSeriesPoint{"sdb": points, "unlisted": points}, err: missing}+ provider := NewLiveProvider(fetcher)+ if err := provider.Refresh(context.Background()); err != nil {+ t.Fatal(err)+ }+ got, err := provider.PhysicalDiskTemperatureHistory(context.Background(), time.Hour)+ if !errors.Is(err, missing) || len(got) != 1 || !reflect.DeepEqual(got["CURRENT-SERIAL"], points) || !reflect.DeepEqual(fetcher.queried, []string{"sdb"}) {+ t.Fatalf("canonical partial identity/error lost: %+v err %v query %+v", got, err, fetcher.queried)+ }+ got["CURRENT-SERIAL"][0].Value = 999+ if points[0].Value != 32.25 {+ t.Fatal("provider history aliases fetcher response")+ }+ fetcher.beforeReturn = func() {+ fetcher.Snapshot.Disks = []Disk{{Name: "sdb", Serial: "REPLACEMENT-SERIAL"}}+ if err := provider.Refresh(context.Background()); err != nil {+ t.Fatal(err)+ }+ }+ if got, err := provider.PhysicalDiskTemperatureHistory(context.Background(), time.Hour); len(got) != 0 || !errors.Is(err, missing) {+ t.Fatalf("in-flight replaced disk served old samples: %+v err %v", got, err)+ }+}++func TestDiskHistoryProviderRejectsAmbiguousAliases(t *testing.T) {+ for _, disks := range [][]Disk{+ {{Name: "sda", Serial: "duplicate"}, {Name: "sdb", Serial: "duplicate"}},+ {{Name: "sda", Serial: "serial-a"}, {Name: "sda", Serial: "serial-b"}},+ } {+ fixtures := DefaultFixtures()+ fixtures.Disks = disks+ fetcher := &diskHistoryResultFetcher{FixtureFetcher: FixtureFetcher{Snapshot: fixtures}, history: map[string][]TimeSeriesPoint{"sda": {{Timestamp: time.Now(), Value: 30}}, "sdb": {{Timestamp: time.Now(), Value: 50}}}}+ provider := NewLiveProvider(fetcher)+ if err := provider.Refresh(context.Background()); err != nil {+ t.Fatal(err)+ }+ if got, err := provider.PhysicalDiskTemperatureHistory(context.Background(), time.Hour); len(got) != 0 || err != nil {+ t.Fatalf("ambiguous inventory attached arbitrary History: %+v err %v", got, err)+ }+ }+}diff --git a/internal/truenas/disk_history_netdata_test.go b/internal/truenas/disk_history_netdata_test.gonew file mode 100644index 0000000000..786c1f95f0--- /dev/null+++ b/internal/truenas/disk_history_netdata_test.go@@ -0,0 +1,151 @@+package truenas++import (+ "context"+ "errors"+ "reflect"+ "sync/atomic"+ "testing"+ "time"++ "github.com/gorilla/websocket"+)++// These are protocol fixtures for #2519's two distinct reporting methods, not+// captures of the reporter's appliance. Epochs/Celsius are actual row samples;+// aggregation means must never be projected as historical observations.+func diskHistoryFixture(identifier string, end int64) map[string]any {+ return map[string]any{+ "name": "disktemp", "identifier": identifier, "legend": []string{"temperature"},+ "data": [][]any{{end - 3540, 31.25}, {end - 1800, 33.5}, {end - 60, 32.75}},+ "aggregations": map[string]any{"mean": []float64{999}},+ }+}++func TestDiskHistoryFillsOnlyMissingSeriesFromNetdata(t *testing.T) {+ var calls atomic.Int32+ var end int64+ server := newMockServerWithRPC(t, defaultAPIResponses(), nil, func(t *testing.T, conn *websocket.Conn) {+ auth := readRPCRequest(t, conn)+ writeRPCResult(t, conn, auth.ID, true)+ primary := readRPCRequest(t, conn)+ calls.Add(1)+ if primary.Method != "reporting.get_data" {+ t.Fatalf("primary method %q", primary.Method)+ }+ params := primary.Params.([]any)+ query := params[1].(map[string]any)+ end = assertReportingRange(t, query, 3600)+ if !reflect.DeepEqual(params[0], []any{map[string]any{"name": "disktemp", "identifier": "sda"}, map[string]any{"name": "disktemp", "identifier": "sdb"}}) {+ t.Fatalf("primary graphs: %#v", params[0])+ }+ writeRPCResult(t, conn, primary.ID, []any{diskHistoryFixture("sda", end)})+ native := readRPCRequest(t, conn)+ calls.Add(1)+ if native.Method != "reporting.netdata_get_data" {+ t.Fatalf("native method %q", native.Method)+ }+ nativeParams := native.Params.([]any)+ if !reflect.DeepEqual(nativeParams[1], query) || !reflect.DeepEqual(nativeParams[0], []any{map[string]any{"name": "disktemp", "identifier": "sdb"}}) {+ t.Fatalf("native read widened window/identities: %#v", nativeParams)+ }+ result := diskHistoryFixture("sdb", end)+ result["data"] = [][]any{{end - 3601, 10}, {end - 3540, 31.25}, {end - 1800, 33.5}, {end - 60, 32.75}, {end + 1, 50}, {end - 50, nil}, {end - 40, 0}, {end - 30, -1}}+ // A response for an unrequested disk must not overwrite the successful+ // primary series or escape the fixed missing-identifier query.+ wrong := diskHistoryFixture("sda", end)+ wrong["data"] = [][]any{{end - 60, 999}}+ writeRPCResult(t, conn, native.ID, []any{result, wrong, diskHistoryFixture("foreign", end)})+ })+ t.Cleanup(server.Close)+ client := mustClientForServer(t, server.URL, ClientConfig{APIKey: "fixture-key"})+ got, err := client.GetDiskTemperatureHistory(context.Background(), []string{"sda", "sdb", "sdb"}, time.Hour)+ if err != nil {+ t.Fatal(err)+ }+ if len(got) != 2 || calls.Load() != 2 {+ t.Fatalf("history/calls: %+v / %d", got, calls.Load())+ }+ want := []TimeSeriesPoint{{Timestamp: time.Unix(end-3540, 0).UTC(), Value: 31.25}, {Timestamp: time.Unix(end-1800, 0).UTC(), Value: 33.5}, {Timestamp: time.Unix(end-60, 0).UTC(), Value: 32.75}}+ for _, identifier := range []string{"sda", "sdb"} {+ if !reflect.DeepEqual(got[identifier], want) {+ t.Fatalf("%s timestamps/Celsius/window changed: %+v", identifier, got[identifier])+ }+ }+}++func TestDiskHistoryPrimaryFailureDoesNotTryNetdata(t *testing.T) {+ for _, test := range []struct {+ name string+ code int+ }{{"EACCES", -32001}, {"EINVAL", -32001}, {"ENOSYS", -32601}} {+ t.Run(test.name, func(t *testing.T) {+ var calls atomic.Int32+ server := newMockServerWithRPC(t, defaultAPIResponses(), nil, func(t *testing.T, conn *websocket.Conn) {+ auth := readRPCRequest(t, conn)+ writeRPCResult(t, conn, auth.ID, true)+ req := readRPCRequest(t, conn)+ calls.Add(1)+ writeReportingRPCFailure(t, conn, req.ID, test.code, test.name, "fixture failure")+ })+ t.Cleanup(server.Close)+ client := mustClientForServer(t, server.URL, ClientConfig{APIKey: "fixture-key"})+ got, err := client.GetDiskTemperatureHistory(context.Background(), []string{"sda"}, time.Hour)+ var rpcErr *RPCError+ if len(got) != 0 || !errors.As(err, &rpcErr) || rpcErr.Method != "reporting.get_data" || rpcErr.Code != test.code || rpcErr.Errname != test.name || calls.Load() != 1 {+ t.Fatalf("failure/fallback changed: history %+v err %v calls %d", got, err, calls.Load())+ }+ })+ }+}++func TestDiskHistoryNetdataFailureKeepsIndependentSeries(t *testing.T) {+ for _, test := range []struct {+ name string+ code int+ }{{"EACCES", -32001}, {"ENOSYS", -32601}, {"EINVAL", -32001}} {+ t.Run(test.name, func(t *testing.T) {+ var calls atomic.Int32+ server := newMockServerWithRPC(t, defaultAPIResponses(), nil, func(t *testing.T, conn *websocket.Conn) {+ auth := readRPCRequest(t, conn)+ writeRPCResult(t, conn, auth.ID, true)+ req := readRPCRequest(t, conn)+ calls.Add(1)+ end := int64(req.Params.([]any)[1].(map[string]any)["end"].(float64))+ writeRPCResult(t, conn, req.ID, []any{diskHistoryFixture("sda", end)})+ req = readRPCRequest(t, conn)+ calls.Add(1)+ writeReportingRPCFailure(t, conn, req.ID, test.code, test.name, "fixture failure")+ })+ t.Cleanup(server.Close)+ client := mustClientForServer(t, server.URL, ClientConfig{APIKey: "fixture-key"})+ got, err := client.GetDiskTemperatureHistory(context.Background(), []string{"sda", "sdb"}, time.Hour)+ var rpcErr *RPCError+ if len(got["sda"]) != 3 || len(got) != 1 || !errors.As(err, &rpcErr) || rpcErr.Method != "reporting.netdata_get_data" || rpcErr.Errname != test.name || calls.Load() != 2 {+ t.Fatalf("partial/failure lost: history %+v err %v calls %d", got, err, calls.Load())+ }+ })+ }+}++func TestDiskHistoryEmptyNetdataDoesNotInventSamples(t *testing.T) {+ var calls atomic.Int32+ server := newMockServerWithRPC(t, defaultAPIResponses(), nil, func(t *testing.T, conn *websocket.Conn) {+ auth := readRPCRequest(t, conn)+ writeRPCResult(t, conn, auth.ID, true)+ for _, method := range []string{"reporting.get_data", "reporting.netdata_get_data"} {+ req := readRPCRequest(t, conn)+ calls.Add(1)+ if req.Method != method {+ t.Fatalf("method %q want %q", req.Method, method)+ }+ writeRPCResult(t, conn, req.ID, []any{})+ }+ })+ t.Cleanup(server.Close)+ client := mustClientForServer(t, server.URL, ClientConfig{APIKey: "fixture-key"})+ got, err := client.GetDiskTemperatureHistory(context.Background(), []string{"sda"}, time.Hour)+ if err != nil || len(got) != 0 || calls.Load() != 2 {+ t.Fatalf("empty result changed: history %+v err %v calls %d", got, err, calls.Load())+ }+}diff --git a/internal/truenas/provider.go b/internal/truenas/provider.goindex f24f0efe4b..e6363ad69e 100644--- a/internal/truenas/provider.go+++ b/internal/truenas/provider.go@@ -394,39 +394,27 @@ func (p *Provider) PhysicalDiskTemperatureHistory(ctx context.Context, durationreturn nil, fmt.Errorf("truenas provider has no cached snapshot")}- identifiers := make([]string, 0, len(snapshot.Disks))- metricIDsByIdentifier := make(map[string]string, len(snapshot.Disks)*3)- for _, disk := range snapshot.Disks {- metricID := trueNASDiskMetricResourceID(disk)- if metricID == "" {- continue- }- if name := strings.TrimSpace(disk.Name); name != "" {- identifiers = append(identifiers, name)- }- for _, key := range trueNASDiskHistoryLookupKeys(disk) {- if _, exists := metricIDsByIdentifier[key]; !exists {- metricIDsByIdentifier[key] = metricID- }- }- }- identifiers = dedupeStrings(identifiers)+ identifiers, metricIDsByIdentifier := trueNASDiskHistoryIdentities(snapshot.Disks)if len(identifiers) == 0 {return nil, nil}nativeHistory, err := historyFetcher.DiskTemperatureHistory(ctx, identifiers, duration)- if err != nil {+ if len(nativeHistory) == 0 {return nil, err}- if len(nativeHistory) == 0 {- return nil, nil+ // Inventory may change while the RPC is in flight. Never attach an old+ // disk's series to a replacement occupying the same native device name.+ currentSnapshot := p.Snapshot()+ if currentSnapshot == nil {+ return nil, err}+ _, currentMetricIDs := trueNASDiskHistoryIdentities(currentSnapshot.Disks)historyByMetricID := make(map[string][]TimeSeriesPoint, len(nativeHistory))for identifier, points := range nativeHistory {metricID := metricIDsByIdentifier[strings.TrimSpace(identifier)]- if metricID == "" || len(points) == 0 {+ if metricID == "" || currentMetricIDs[strings.TrimSpace(identifier)] != metricID || len(points) == 0 {continue}copied := make([]TimeSeriesPoint, len(points))@@ -434,9 +422,35 @@ func (p *Provider) PhysicalDiskTemperatureHistory(ctx context.Context, durationhistoryByMetricID[metricID] = copied}if len(historyByMetricID) == 0 {- return nil, nil+ return nil, err+ }+ return historyByMetricID, err+}++func trueNASDiskHistoryIdentities(disks []Disk) ([]string, map[string]string) {+ identifiers := make([]string, 0, len(disks))+ metricIDs := make(map[string]string, len(disks)*3)+ counts := make(map[string]int, len(disks))+ for _, disk := range disks {+ counts[trueNASDiskMetricResourceID(disk)]+++ }+ for _, disk := range disks {+ metricID := trueNASDiskMetricResourceID(disk)+ if metricID == "" || counts[metricID] != 1 {+ continue+ }+ if name := strings.TrimSpace(disk.Name); name != "" {+ identifiers = append(identifiers, name)+ }+ for _, key := range trueNASDiskHistoryLookupKeys(disk) {+ if existing, exists := metricIDs[key]; exists && existing != metricID {+ metricIDs[key] = "" // ambiguous aliases stay unusable+ } else if !exists {+ metricIDs[key] = metricID+ }+ }}- return historyByMetricID, nil+ return dedupeStrings(identifiers), metricIDs}// Close releases resources held by the active fetcher, if supported.