Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Keep native host I/O visible beside stored CPU history
pulse-triage[bot]495c0cc6ab19
diff --git a/internal/api/router.go b/internal/api/router.goindex 424602b758..3338fbfc9d 100644--- a/internal/api/router.go+++ b/internal/api/router.go@@ -6816,7 +6816,7 @@ func (r *Router) handleMetricsHistory(w http.ResponseWriter, req *http.Request)return points}- guestChartMetrics := func() (map[string][]monitoring.MetricPoint, bool) {+ guestChartMetrics := func(metricTypes ...string) (map[string][]monitoring.MetricPoint, bool) {inMemoryKey := resourceIDsqlResourceType := ""switch runtimeResourceType {@@ -6838,7 +6838,7 @@ func (r *Router) handleMetricsHistory(w http.ResponseWriter, req *http.Request)default:return nil, false}- return monitor.GetGuestMetricsForChart(inMemoryKey, sqlResourceType, resourceID, duration), true+ return monitor.GetGuestMetricsForChart(inMemoryKey, sqlResourceType, resourceID, duration, metricTypes...), true}guestChartSource := func() string {if mock.IsMockEnabled() {@@ -6893,7 +6893,7 @@ func (r *Router) handleMetricsHistory(w http.ResponseWriter, req *http.Request)switch runtimeResourceType {case "vm", "system-container", "oci-container":- metrics, _ := guestChartMetrics()+ metrics, _ := guestChartMetrics(metricType)points := metrics[metricType]if len(points) == 0 {livePoints := liveMetricPoints(runtimeResourceType, resourceID)@@ -6904,7 +6904,7 @@ func (r *Router) handleMetricsHistory(w http.ResponseWriter, req *http.Request)}return buildHistoryPoints(points, stepSecs), guestChartSource(), truecase "docker-host":- metrics, _ := guestChartMetrics()+ metrics, _ := guestChartMetrics(metricType)points := metrics[metricType]if len(points) == 0 {livePoints := liveMetricPoints(runtimeResourceType, resourceID)@@ -6915,7 +6915,7 @@ func (r *Router) handleMetricsHistory(w http.ResponseWriter, req *http.Request)}return buildHistoryPoints(points, stepSecs), guestChartSource(), truecase "agent":- metrics, _ := guestChartMetrics()+ metrics, _ := guestChartMetrics(metricType)points := metrics[metricType]if len(points) == 0 {points = monitor.GetNodeMetrics(resourceID, metricType, duration)@@ -6929,7 +6929,7 @@ func (r *Router) handleMetricsHistory(w http.ResponseWriter, req *http.Request)}return buildHistoryPoints(points, stepSecs), guestChartSource(), truecase "app-container":- metrics, _ := guestChartMetrics()+ metrics, _ := guestChartMetrics(metricType)points := metrics[metricType]if len(points) == 0 {livePoints := liveMetricPoints(runtimeResourceType, resourceID)@@ -6940,7 +6940,7 @@ func (r *Router) handleMetricsHistory(w http.ResponseWriter, req *http.Request)}return buildHistoryPoints(points, stepSecs), guestChartSource(), truecase "k8s":- metrics, _ := guestChartMetrics()+ metrics, _ := guestChartMetrics(metricType)points := metrics[metricType]if len(points) == 0 {livePoints := liveMetricPoints(runtimeResourceType, resourceID)@@ -7404,6 +7404,33 @@ func (r *Router) handleMetricsHistory(w http.ResponseWriter, req *http.Request)}}+ // A stored CPU/capacity row does not establish host I/O coverage.+ // Ask only for missing directions; retain every stored series and+ // the native parser's complete-member/coincident-time requirement.+ if runtimeResourceType == "agent" && !mock.IsMockEnabled() {+ var missing []string+ for _, metric := range []string{"diskread", "diskwrite"} {+ if len(apiData[metric]) == 0 {+ missing = append(missing, metric)+ }+ }+ if len(missing) > 0 {+ fallback, _ := guestChartMetrics(missing...)+ for _, metric := range missing {+ var withinWindow []monitoring.MetricPoint+ for _, point := range fallback[metric] {+ if !point.Timestamp.Before(start) && !point.Timestamp.After(end) {+ withinWindow = append(withinWindow, point)+ }+ }+ if len(withinWindow) > 0 {+ apiData[metric] = buildHistoryPoints(withinWindow, stepSecs)+ source = historySourceMemory+ }+ }+ }+ }+// QueryAll can return a non-empty but incomplete mock metric map. Fill// only the absent series from the deterministic chart fallback so one// populated metric cannot suppress entire drawer groups.diff --git a/internal/api/truenas_host_io_history_test.go b/internal/api/truenas_host_io_history_test.gonew file mode 100644index 0000000000..29950fd786--- /dev/null+++ b/internal/api/truenas_host_io_history_test.go@@ -0,0 +1,303 @@+package api++import (+ "context"+ "crypto/sha256"+ "encoding/json"+ "fmt"+ "net/http"+ "net/http/httptest"+ "os"+ "path/filepath"+ "sync/atomic"+ "testing"+ "time"++ "github.com/rcourtman/pulse-go-rewrite/internal/config"+ "github.com/rcourtman/pulse-go-rewrite/internal/models"+ "github.com/rcourtman/pulse-go-rewrite/internal/monitoring"+ "github.com/rcourtman/pulse-go-rewrite/internal/truenas"+ "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"+ "github.com/rcourtman/pulse-go-rewrite/pkg/metrics"+)++// This fixture models CORE's REST graph contract, not a reporter's appliance.+// Live disjoint disk samples survive individually but cannot form a host sum.+// History phases independently control whether a complete native sum exists.+type hostIORESTFixture struct {+ server *httptest.Server+ phase atomic.Int32+ reads atomic.Int32+ end atomic.Int64+}++func newHostIORESTFixture(t *testing.T) *hostIORESTFixture {+ t.Helper()+ f := &hostIORESTFixture{}+ f.server = httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ switch r.URL.Path {+ case "/api/v2.0/system/info":+ fmt.Fprint(w, `{"hostname":"same-core-name","version":"TrueNAS-13.0-U6.8","cores":8,"physmem":17179869184}`)+ case "/api/v2.0/pool":+ fmt.Fprint(w, `[{"id":1,"name":"tank","status":"ONLINE","size":1000,"allocated":400,"free":600}]`)+ case "/api/v2.0/disk":+ fmt.Fprint(w, `[{"identifier":"disk-a","name":"da0","serial":"host-io-a","type":"HDD","size":1000},{"identifier":"disk-b","name":"da1","serial":"host-io-b","type":"HDD","size":1000}]`)+ case "/api/v2.0/pool/dataset", "/api/v2.0/alert/list":+ fmt.Fprint(w, `[]`)+ case "/api/v2.0/reporting/graphs":+ if f.phase.Load() == 2 {+ fmt.Fprint(w, `[{"name":"disk","identifiers":["da0","da1","da2"]}]`)+ } else {+ fmt.Fprint(w, `[{"name":"disk","identifiers":["da0","da1"]}]`)+ }+ case "/api/v2.0/reporting/get_data":+ f.reads.Add(1)+ var request struct {+ Graphs []struct{ Name, Identifier string }+ Query struct{ Start, End int64 } `json:"reporting_query"`+ }+ if r.Method != http.MethodPost || json.NewDecoder(r.Body).Decode(&request) != nil || request.Query.End <= request.Query.Start {+ http.Error(w, "invalid synthetic reporting request", 400)+ return+ }+ f.end.Store(request.Query.End)+ var responses []map[string]any+ for _, graph := range request.Graphs {+ start, step := request.Query.End-3540, int64(1160)+ var legend []string+ var row []any+ switch graph.Name {+ case "cpu":+ legend, row = []string{"usage"}, []any{12.}+ case "disk":+ legend, row = []string{"disk_octets_read", "disk_octets_write"}, []any{100., 200.}+ if graph.Identifier == "da1" {+ row = []any{1000., 2000.}+ }+ if f.phase.Load() == 3 {+ row = []any{0., 0.}+ }+ if f.phase.Load() == 4 && graph.Identifier == "da1" {+ row[0] = nil+ }+ default:+ continue+ }+ rows := []any{row, row, row, row}+ if graph.Identifier == "da2" {+ rows = nil+ }+ if f.phase.Load() == 0 && graph.Name == "disk" {+ start, step, rows = request.Query.End-20, 10, []any{row, row}+ if graph.Identifier == "da1" {+ start += 5+ }+ }+ responses = append(responses, map[string]any{"name": graph.Name, "identifier": graph.Identifier, "legend": legend, "start": start, "end": request.Query.End, "step": step, "data": rows})+ }+ if err := json.NewEncoder(w).Encode(responses); err != nil {+ t.Error(err)+ }+ default:+ http.NotFound(w, r)+ }+ }))+ t.Cleanup(f.server.Close)+ return f+}++func (f *hostIORESTFixture) provider(t *testing.T, connection string) *truenas.Provider {+ t.Helper()+ client, err := truenas.NewClient(truenas.ClientConfig{Host: f.server.URL, APIKey: "synthetic-host-io-key", InsecureSkipVerify: true, Fingerprint: fmt.Sprintf("%x", sha256.Sum256(f.server.Certificate().Raw))})+ if err != nil {+ t.Fatal(err)+ }+ t.Cleanup(client.Close)+ p := truenas.NewLiveProviderForConnection(&truenas.APIFetcher{Client: client}, connection)+ if err := p.Refresh(context.Background()); err != nil {+ t.Fatal(err)+ }+ return p+}++type hostIOHistoryPoint struct {+ Timestamp int64 `json:"timestamp"`+ Value float64 `json:"value"`+ Min float64 `json:"min"`+ Max float64 `json:"max"`+}++// Exercise native CORE collection -> canonical ingestion -> persisted HTTP,+// including the drawer's all-metric query with unrelated covered CPU history.+func TestTrueNASHostIOHistoryWithPartialStore(t *testing.T) {+ previous := truenas.IsFeatureEnabled()+ truenas.SetFeatureEnabled(true)+ t.Cleanup(func() { truenas.SetFeatureEnabled(previous) })+ previousMulti := IsMultiTenantEnabled()+ SetMultiTenantEnabled(true)+ t.Cleanup(func() { SetMultiTenantEnabled(previousMulti) })+ t.Setenv("PULSE_DEV", "true")+ f, other := newHostIORESTFixture(t), newHostIORESTFixture(t)+ provider, otherProvider := f.provider(t, "host-io-target"), other.provider(t, "other-target")+ snapshot := provider.Snapshot()+ if snapshot.System.Telemetry.DiskRead || snapshot.System.Telemetry.DiskWrite || len(snapshot.System.DiskReadRates) != 2 {+ t.Fatal("fixture did not establish complete per-disk readings with an absent host total")+ }+ store, err := metrics.NewStore(metrics.DefaultConfig(t.TempDir()))+ if err != nil {+ t.Fatal(err)+ }+ t.Cleanup(func() { _ = store.Close() })+ poller := monitoring.NewTrueNASPoller(nil, time.Minute, nil)+ setUnexportedField(t, poller, "providersByOrg", map[string]map[string]*truenas.Provider{"org-host-io": {"host-io-target": provider, "other-target": otherProvider}})+ setUnexportedField(t, poller, "cachedRecordsByOrg", map[string]map[string][]unifiedresources.IngestRecord{"org-host-io": {"host-io-target": provider.Records()}})+ monitor := &monitoring.Monitor{}+ monitor.SetOrgID("org-host-io")+ setUnexportedField(t, monitor, "state", models.NewState())+ ring := monitoring.NewMetricsHistory(1024, 24*time.Hour)+ setUnexportedField(t, monitor, "metricsHistory", ring)+ setUnexportedField(t, monitor, "metricsStore", store)+ monitor.SetResourceStoreAndProviders(unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil)), map[unifiedresources.DataSource]monitoring.MonitorSupplementalRecordsProvider{unifiedresources.SourceTrueNAS: poller})+ store.Flush()+ now := time.Now().UTC().Truncate(time.Second)+ // Simulate an established CPU ring. That must not certify an absent I/O series.+ for _, offset := range []time.Duration{-55 * time.Minute, -30 * time.Minute, -time.Minute} {+ ring.AddGuestMetric("agent:host-io-target", "cpu", 19, now.Add(offset))+ }+ const token = "host-io-reader.1234567890123456789"+ readToken := newTokenRecord(t, token, []string{config.ScopeMonitoringRead}, nil)+ readToken.OrgID = "org-host-io"+ const wrongScope = "host-io-settings.1234567890123456789"+ scopeToken := newTokenRecord(t, wrongScope, []string{config.ScopeSettingsRead}, nil)+ scopeToken.OrgID = "org-host-io"+ cfg := newTestConfigWithTokens(t, readToken, scopeToken)+ cfg.AuthUser, cfg.AuthPass = "fixture-admin", "synthetic-unused-password-hash"+ for _, org := range []string{"org-host-io", "org-other"} {+ if err := os.MkdirAll(filepath.Join(cfg.DataPath, "orgs", org), 0700); err != nil {+ t.Fatal(err)+ }+ }+ router := NewRouter(cfg, monitor, nil, nil, nil, "test")+ t.Cleanup(router.ShutdownBackgroundWorkers)+ t.Cleanup(router.ShutdownResourceStores)+ t.Cleanup(router.ShutdownRBAC)+ mtm := monitoring.NewMultiTenantMonitor(cfg, nil, nil)+ setUnexportedField(t, mtm, "monitors", map[string]*monitoring.Monitor{"org-host-io": monitor, "org-other": {}})+ router.mtMonitor = mtm+ router.resourceHandlers.SetTenantStateProvider(NewMultiTenantStateProvider(mtm, monitor))+ router.setMonitorSupplementalRecordsProvider(unifiedresources.SourceTrueNAS, poller)+ request := func(raw, org, resourceType, id, metric string) *httptest.ResponseRecorder {+ req := httptest.NewRequest(http.MethodGet, "/api/metrics-store/history?resourceType="+resourceType+"&resourceId="+id+"&metric="+metric+"&range=1h&maxPoints=240", nil)+ req.Header.Set("X-Pulse-Org-ID", org)+ if raw != "" {+ req.Header.Set("X-API-Token", raw)+ }+ rec := httptest.NewRecorder()+ router.Handler().ServeHTTP(rec, req)+ return rec+ }+ f.reads.Store(0)+ other.reads.Store(0)+ for _, test := range []struct {+ raw, org string+ want int+ }{{"", "org-host-io", 401}, {wrongScope, "org-host-io", 403}, {token, "org-other", 403}} {+ if rec := request(test.raw, test.org, "agent", "host-io-target", ""); rec.Code != test.want || f.reads.Load() != 0 || other.reads.Load() != 0 {+ t.Fatalf("auth/scope/tenant gate: status %d want %d reads %d/%d", rec.Code, test.want, f.reads.Load(), other.reads.Load())+ }+ }+ // Actual per-disk persistence is independent of the native host fallback.+ for _, serial := range []string{"host-io-a", "host-io-b"} {+ rec := request(token, "org-host-io", "disk", serial, "diskread")+ var result struct{ Points []hostIOHistoryPoint }+ if rec.Code != 200 || json.Unmarshal(rec.Body.Bytes(), &result) != nil || len(result.Points) == 0 {+ t.Fatalf("individual disk lost persisted I/O: %d %s", rec.Code, rec.Body.String())+ }+ }+ if f.reads.Load() != 0 || other.reads.Load() != 0 {+ t.Fatal("persisted per-disk I/O queried native host History")+ }+ var storedCPU []hostIOHistoryPoint+ for _, test := range []struct {+ name string+ phase int32+ read, write bool+ value float64+ }{{"complete", 1, true, true, 1100}, {"empty-member", 2, false, false, 0}, {"measured-zero", 3, true, true, 0}, {"write-only", 4, false, true, 0}, {"disjoint", 0, false, false, 0}} {+ t.Run(test.name, func(t *testing.T) {+ f.phase.Store(test.phase)+ before := f.reads.Load()+ rec := request(token, "org-host-io", "agent", "host-io-target", "")+ var result struct {+ Metrics map[string][]hostIOHistoryPoint+ }+ if rec.Code != 200 || json.Unmarshal(rec.Body.Bytes(), &result) != nil {+ t.Fatalf("host history %d %s", rec.Code, rec.Body.String())+ }+ if (len(result.Metrics["diskread"]) > 0) != test.read || (len(result.Metrics["diskwrite"]) > 0) != test.write {+ t.Fatalf("host I/O completeness: read=%d write=%d want %t/%t", len(result.Metrics["diskread"]), len(result.Metrics["diskwrite"]), test.read, test.write)+ }+ if f.reads.Load()-before != 1 || other.reads.Load() != 0 {+ t.Fatalf("host fallback read other appliance or lost its one-request bound: reads %d/%d", f.reads.Load()-before, other.reads.Load())+ }+ if len(result.Metrics["cpu"]) != 1 || result.Metrics["cpu"][0].Value != 12 {+ t.Fatalf("host fallback replaced stored CPU: %+v", result.Metrics["cpu"])+ }+ if storedCPU == nil {+ storedCPU = result.Metrics["cpu"]+ } else if result.Metrics["cpu"][0] != storedCPU[0] {+ t.Fatal("native supplement changed stored CPU time/bounds")+ }+ for _, metric := range []string{"diskread", "diskwrite"} {+ for i, point := range result.Metrics[metric] {+ value := test.value+ if metric == "diskwrite" && test.phase != 3 {+ value = 2200+ }+ ts := f.end.Load() - 3540 + int64(i)*1160+ if point.Value != value || point.Min != value || point.Max != value || point.Timestamp != (ts/15*15+7)*1000 {+ t.Fatalf("native rate/time/bucket/bounds changed: %s %+v", metric, point)+ }+ }+ }+ t.Logf("HOST_IO_BROWSER %s %s", test.name, rec.Body.String())+ })+ }+ // Single-series reads must qualify that series, not a covered CPU sibling.+ f.phase.Store(1)+ before := f.reads.Load()+ rec := request(token, "org-host-io", "agent", "host-io-target", "diskread")+ var single struct{ Points []hostIOHistoryPoint }+ if rec.Code != 200 || json.Unmarshal(rec.Body.Bytes(), &single) != nil || len(single.Points) != 4 || f.reads.Load()-before != 1 || other.reads.Load() != 0 {+ t.Fatalf("single host I/O hidden by covered CPU: status %d points %d reads %d/%d", rec.Code, len(single.Points), f.reads.Load()-before, other.reads.Load())+ }+ // Existing stored directions are not replaced by a larger native series.+ ioTime := now.Add(-20 * time.Second)+ store.WriteBatchSync([]metrics.WriteMetric{{ResourceType: "agent", ResourceID: "host-io-target", MetricType: "diskread", Value: 777, Timestamp: ioTime, Tier: metrics.TierRaw}})+ rec = request(token, "org-host-io", "agent", "host-io-target", "")+ var partial struct {+ Metrics map[string][]hostIOHistoryPoint+ }+ if rec.Code != 200 || json.Unmarshal(rec.Body.Bytes(), &partial) != nil || len(partial.Metrics["diskread"]) != 1 || partial.Metrics["diskread"][0].Value != 777 || len(partial.Metrics["diskwrite"]) != 4 {+ t.Fatalf("native fill replaced stored direction: %s", rec.Body.String())+ }+ store.WriteBatchSync([]metrics.WriteMetric{{ResourceType: "agent", ResourceID: "host-io-target", MetricType: "diskwrite", Value: 888, Timestamp: ioTime, Tier: metrics.TierRaw}})+ before = f.reads.Load()+ rec = request(token, "org-host-io", "agent", "host-io-target", "")+ if rec.Code != 200 || f.reads.Load() != before || other.reads.Load() != 0 {+ t.Fatal("complete stored I/O needlessly queried native History")+ }+ if err := store.Close(); err != nil {+ t.Fatal(err)+ }+ if rec := request(token, "org-host-io", "agent", "host-io-target", ""); rec.Code != 500 || f.reads.Load() != before {+ t.Fatal("failed store read became successful native history")+ }+ if cfg.RemoveAPIToken(readToken.ID) == nil {+ t.Fatal("missing fixture token")+ }+ if rec := request(token, "org-host-io", "agent", "host-io-target", ""); rec.Code != 401 || f.reads.Load() != before {+ t.Fatal("withdrawn reader reached native history")+ }+}diff --git a/internal/monitoring/monitor_metrics.go b/internal/monitoring/monitor_metrics.goindex e52b8e9c34..03d80c145b 100644--- a/internal/monitoring/monitor_metrics.go+++ b/internal/monitoring/monitor_metrics.go@@ -72,6 +72,12 @@ type MonitorGuestMetricHistoryProvider interface {GuestMetricHistory(m *Monitor, orgID string, resourceType string, duration time.Duration) map[string]map[string][]MetricPoint}+// MonitorTargetedGuestMetricHistoryProvider avoids querying other appliances+// when a single resource needs a source-native chart fallback.+type MonitorTargetedGuestMetricHistoryProvider interface {+ GuestMetricHistoryForResource(m *Monitor, orgID, resourceType, resourceID string, duration time.Duration) map[string][]MetricPoint+}+// GetGuestMetrics returns historical metrics for a guestfunc (m *Monitor) GetGuestMetrics(guestID string, duration time.Duration) map[string][]MetricPoint {if m == nil || m.metricsHistory == nil {@@ -106,17 +112,18 @@ func (m *Monitor) GetGuestMetrics(guestID string, duration time.Duration) map[st// inMemoryKey is the key used in the in-memory buffer (e.g. "docker:abc123").// sqlResourceType/sqlResourceID are the type/id used in the SQLite store// (e.g. "dockerContainer"/"abc123").-func (m *Monitor) GetGuestMetricsForChart(inMemoryKey, sqlResourceType, sqlResourceID string, duration time.Duration) map[string][]MetricPoint {+func (m *Monitor) GetGuestMetricsForChart(inMemoryKey, sqlResourceType, sqlResourceID string, duration time.Duration, metricTypes ...string) map[string][]MetricPoint {+ requestedMetricTypes := normalizeChartMetricTypes(metricTypes)if mock.IsMockEnabled() {- return m.mockGuestMetricsForChart(inMemoryKey, sqlResourceType, sqlResourceID, duration, m.GetGuestMetrics(inMemoryKey, duration))+ return filterChartMetricMap(m.mockGuestMetricsForChart(inMemoryKey, sqlResourceType, sqlResourceID, duration, m.GetGuestMetrics(inMemoryKey, duration)), requestedMetricTypes)}inMemoryResult := map[string][]MetricPoint{}- if m.shouldLoadGuestChartInMemory(inMemoryKey, nil, duration) {- inMemoryResult = m.GetGuestMetrics(inMemoryKey, duration)+ if m.shouldLoadGuestChartInMemory(inMemoryKey, requestedMetricTypes, duration) {+ inMemoryResult = filterChartMetricMap(m.GetGuestMetrics(inMemoryKey, duration), requestedMetricTypes)}- if hasSufficientChartMapCoverage(inMemoryResult, duration) {+ if hasSufficientChartMapCoverageForMetrics(inMemoryResult, duration, requestedMetricTypes) {return inMemoryResult}@@ -125,12 +132,15 @@ func (m *Monitor) GetGuestMetricsForChart(inMemoryKey, sqlResourceType, sqlResoubest := inMemoryResultif m.metricsStore != nil {if converted, ok := m.queryStoreMetricMapWithGapFillAliases(sqlResourceType, sqlResourceID, duration); ok {- best = mergeGuestMetricHistory(best, converted, duration)+ best = mergeGuestMetricHistory(best, filterChartMetricMap(converted, requestedMetricTypes), duration)}}- if nativeHistory := m.nativeGuestMetricHistory(sqlResourceType, duration); len(nativeHistory) > 0 {+ if len(requestedMetricTypes) > 0 && hasSufficientChartMapCoverageForMetrics(best, duration, requestedMetricTypes) {+ return best+ }+ if nativeHistory := m.nativeGuestMetricHistory(sqlResourceType, duration, sqlResourceID); len(nativeHistory) > 0 {if nativeMetrics, ok := nativeHistory[sqlResourceID]; ok {- best = mergeGuestMetricHistory(best, nativeMetrics, duration)+ best = mergeGuestMetricHistory(best, filterChartMetricMap(nativeMetrics, requestedMetricTypes), duration)}}return best@@ -806,7 +816,7 @@ func (m *Monitor) GetStorageMetricsForChartBatch(return result}-func (m *Monitor) nativeGuestMetricHistory(resourceType string, duration time.Duration) map[string]map[string][]MetricPoint {+func (m *Monitor) nativeGuestMetricHistory(resourceType string, duration time.Duration, resourceIDs ...string) map[string]map[string][]MetricPoint {providers := m.supplementalProviderSnapshot()if len(providers) == 0 {return nil@@ -821,6 +831,14 @@ func (m *Monitor) nativeGuestMetricHistory(resourceType string, duration time.Duhistory := make(map[string]map[string][]MetricPoint)for _, provider := range providers {+ if targeted, ok := provider.(MonitorTargetedGuestMetricHistoryProvider); ok && len(resourceIDs) == 1 {+ id := resourceIDs[0]+ metricMap := targeted.GuestMetricHistoryForResource(m, orgID, resourceType, id, duration)+ if len(metricMap) > 0 {+ history[id] = mergeGuestMetricHistory(history[id], metricMap, duration)+ }+ continue+ }historyProvider, ok := provider.(MonitorGuestMetricHistoryProvider)if !ok {continuediff --git a/internal/monitoring/monitor_metrics_requested_history_test.go b/internal/monitoring/monitor_metrics_requested_history_test.gonew file mode 100644index 0000000000..703d75d5ba--- /dev/null+++ b/internal/monitoring/monitor_metrics_requested_history_test.go@@ -0,0 +1,57 @@+package monitoring++import (+ "testing"+ "time"++ "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"+)++type targetedChartHistoryFixture struct {+ stubGuestMetricHistoryProvider+ reads, bulkReads int+ id, org, kind string+}++func (s *targetedChartHistoryFixture) GuestMetricHistory(*Monitor, string, string, time.Duration) map[string]map[string][]MetricPoint {+ s.bulkReads+++ return nil+}++func (s *targetedChartHistoryFixture) GuestMetricHistoryForResource(_ *Monitor, org, kind, id string, _ time.Duration) map[string][]MetricPoint {+ s.reads+++ s.id, s.org, s.kind = id, org, kind+ return s.stubGuestMetricHistoryProvider.GuestMetricHistory(nil, org, kind, time.Hour)[id]+}++func TestGuestChartRequestedCoverageIsPerSeries(t *testing.T) {+ t.Parallel()+ for _, duration := range []time.Duration{time.Hour, 24 * time.Hour} {+ t.Run(duration.String(), func(t *testing.T) {+ monitor := newChartFallbackTestMonitor(t)+ monitor.SetOrgID("org-targeted")+ now := time.Now().UTC().Truncate(time.Second)+ points := []MetricPoint{{Timestamp: now.Add(-duration + time.Minute), Value: 100}, {Timestamp: now.Add(-time.Minute), Value: 0}}+ provider := &targetedChartHistoryFixture{stubGuestMetricHistoryProvider: stubGuestMetricHistoryProvider{history: map[string]map[string][]MetricPoint{+ "host-target": {"diskread": points, "diskwrite": points, "cpu": {{Timestamp: now, Value: 999}}},+ }}}+ monitor.supplementalProviders = map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider{unifiedresources.SourceTrueNAS: provider}+ for _, point := range points {+ monitor.metricsHistory.AddGuestMetric("agent:host-target", "cpu", 12, point.Timestamp)+ }+ result := monitor.GetGuestMetricsForChart("agent:host-target", "agent", "host-target", duration, "diskread", "diskwrite")+ if len(result["diskread"]) != 2 || len(result["diskwrite"]) != 2 || result["diskread"][1].Value != 0 || len(result["cpu"]) != 0 || provider.reads != 1 || provider.bulkReads != 0 {+ t.Fatalf("covered CPU hid requested IO or target escaped: %+v reads %d/%d", result, provider.reads, provider.bulkReads)+ }+ if provider.id != "host-target" || provider.org != "org-targeted" || provider.kind != "agent" {+ t.Fatalf("target binding changed: %+v", provider)+ }+ writeRawMetricBatch(t, monitor.metricsStore, "agent", "host-target", "diskread", points)+ writeRawMetricBatch(t, monitor.metricsStore, "agent", "host-target", "diskwrite", points)+ monitor.GetGuestMetricsForChart("agent:host-target", "agent", "host-target", duration, "diskread", "diskwrite")+ if provider.reads != 1 {+ t.Fatal("covered requested store series queried native provider")+ }+ })+ }+}diff --git a/internal/monitoring/truenas_host_history_target_test.go b/internal/monitoring/truenas_host_history_target_test.gonew file mode 100644index 0000000000..1ffef8c7e2--- /dev/null+++ b/internal/monitoring/truenas_host_history_target_test.go@@ -0,0 +1,82 @@+package monitoring++import (+ "context"+ "sync/atomic"+ "testing"+ "time"++ "github.com/rcourtman/pulse-go-rewrite/internal/truenas"+)++type targetedTrueNASHistoryFetcher struct {+ controllableTrueNASHistoryFetcher+ reads atomic.Int32+ entered chan struct{}+ release chan struct{}+}++func (f *targetedTrueNASHistoryFetcher) SystemMetricHistory(ctx context.Context, duration time.Duration) (*truenas.SystemMetricHistory, error) {+ f.reads.Add(1)+ if f.entered != nil {+ close(f.entered)+ <-f.release+ }+ return f.controllableTrueNASHistoryFetcher.SystemMetricHistory(ctx, duration)+}++func TestTrueNASHostHistoryTargetAndLifetime(t *testing.T) {+ previous := truenas.IsFeatureEnabled()+ truenas.SetFeatureEnabled(true)+ t.Cleanup(func() { truenas.SetFeatureEnabled(previous) })+ newProvider := func(connection string) (*truenas.Provider, *targetedTrueNASHistoryFetcher) {+ fixture := truenas.DefaultFixtures()+ fixture.System.Hostname = "same-name"+ f := &targetedTrueNASHistoryFetcher{controllableTrueNASHistoryFetcher: controllableTrueNASHistoryFetcher{snapshot: &fixture, systemHistory: &truenas.SystemMetricHistory{DiskReadRate: []truenas.TimeSeriesPoint{{Timestamp: time.Now().Add(-time.Minute), Value: 0}}}}}+ p := truenas.NewLiveProviderForConnection(f, connection)+ if err := p.Refresh(context.Background()); err != nil {+ t.Fatal(err)+ }+ return p, f+ }+ target, targetFetcher := newProvider("target")+ other, otherFetcher := newProvider("other")+ poller := NewTrueNASPoller(nil, time.Minute, nil)+ poller.providersByOrg["org-a"] = map[string]*truenas.Provider{"target": target, "other": other}+ for _, test := range []struct{ org, kind, id string }{{"org-b", "agent", "target"}, {"org-a", "disk", "target"}, {"org-a", "agent", "missing"}, {"org-a", "agent", "same-name"}, {"org-a", "agent", ""}} {+ if result := poller.GuestMetricHistoryForResource(nil, test.org, test.kind, test.id, time.Hour); len(result) != 0 || targetFetcher.reads.Load() != 0 || otherFetcher.reads.Load() != 0 {+ t.Fatalf("nonmatching target reached native history: %+v", test)+ }+ }+ result := poller.GuestMetricHistoryForResource(nil, "org-a", "agent", "target", time.Hour)+ if len(result["diskread"]) != 1 || result["diskread"][0].Value != 0 || targetFetcher.reads.Load() != 1 || otherFetcher.reads.Load() != 0 {+ t.Fatalf("exact target/zero lost or other same-name host read: %+v", result)+ }+ for _, action := range []string{"remove", "replace", "disable"} {+ t.Run(action, func(t *testing.T) {+ p, f := newProvider("target")+ f.entered, f.release = make(chan struct{}), make(chan struct{})+ poller.mu.Lock()+ poller.providersByOrg["org-a"]["target"] = p+ poller.mu.Unlock()+ truenas.SetFeatureEnabled(true)+ done := make(chan map[string][]MetricPoint, 1)+ go func() { done <- poller.GuestMetricHistoryForResource(nil, "org-a", "agent", "target", time.Hour) }()+ <-f.entered+ poller.mu.Lock()+ switch action {+ case "remove":+ delete(poller.providersByOrg["org-a"], "target")+ case "replace":+ poller.providersByOrg["org-a"]["target"] = target+ case "disable":+ truenas.SetFeatureEnabled(false)+ }+ poller.mu.Unlock()+ close(f.release)+ if result := <-done; len(result) != 0 {+ t.Fatalf("withdrawn provider returned native history: %+v", result)+ }+ })+ }+}diff --git a/internal/monitoring/truenas_poller.go b/internal/monitoring/truenas_poller.goindex c620be45ff..8ad2b55f40 100644--- a/internal/monitoring/truenas_poller.go+++ b/internal/monitoring/truenas_poller.go@@ -1075,6 +1075,20 @@ func (p *TrueNASPoller) PhysicalDiskTemperatureHistory(_ *Monitor, orgID string,// GuestMetricHistory exposes native TrueNAS host history through the canonical// guest-chart boundary when local Pulse history is shallow.func (p *TrueNASPoller) GuestMetricHistory(_ *Monitor, orgID string, resourceType string, duration time.Duration) map[string]map[string][]MetricPoint {+ return p.guestMetricHistory(orgID, resourceType, "", duration)+}++// GuestMetricHistoryForResource selects the exact tenant-local host before any+// native read. A missing host must not turn a drawer request into a fleet scan.+func (p *TrueNASPoller) GuestMetricHistoryForResource(_ *Monitor, orgID, resourceType, resourceID string, duration time.Duration) map[string][]MetricPoint {+ resourceID = strings.TrimSpace(resourceID)+ if resourceID == "" {+ return nil+ }+ return p.guestMetricHistory(orgID, resourceType, resourceID, duration)[resourceID]+}++func (p *TrueNASPoller) guestMetricHistory(orgID, resourceType, selectedID string, duration time.Duration) map[string]map[string][]MetricPoint {if p == nil || !truenas.IsFeatureEnabled() || strings.TrimSpace(resourceType) != "agent" {return nil}@@ -1094,6 +1108,9 @@ func (p *TrueNASPoller) GuestMetricHistory(_ *Monitor, orgID string, resourceTyphistory := make(map[string]map[string][]MetricPoint)for _, entry := range entries {+ if selectedID != "" && entry.provider.SystemMetricResourceID() != selectedID {+ continue+ }resourceID, nativeHistory, err := entry.provider.SystemMetricHistory(ctx, duration)if err != nil {log.Warn().@@ -1107,7 +1124,14 @@ func (p *TrueNASPoller) GuestMetricHistory(_ *Monitor, orgID string, resourceTypcontinue}resourceID = strings.TrimSpace(resourceID)- if resourceID == "" || len(nativeHistory) == 0 {+ if resourceID == "" || len(nativeHistory) == 0 || (selectedID != "" && resourceID != selectedID) {+ continue+ }+ // Do not return a provider removed or replaced while its read was in flight.+ p.mu.Lock()+ current := truenas.IsFeatureEnabled() && p.providersByOrg[orgID][entry.connectionID] == entry.provider+ p.mu.Unlock()+ if !current {continue}diff --git a/internal/truenas/provider.go b/internal/truenas/provider.goindex 991534bfcf..9f1cfa268e 100644--- a/internal/truenas/provider.go+++ b/internal/truenas/provider.go@@ -324,6 +324,20 @@ func (p *Provider) GetAppConfig(_ context.Context, appID string) (*AppConfigResureturn result, nil}+// SystemMetricResourceID reports the same native host identity as Records and+// SystemMetricHistory, without starting a provider request.+func (p *Provider) SystemMetricResourceID() string {+ if p == nil {+ return ""+ }+ p.mu.Lock()+ defer p.mu.Unlock()+ if p.lastSnapshot == nil {+ return ""+ }+ return trueNASSystemMetricResourceID(p.connectionID, p.lastSnapshot.System)+}+// SystemMetricHistory returns canonical host-chart metrics for the TrueNAS// appliance keyed by the shared agent metrics resource ID.func (p *Provider) SystemMetricHistory(ctx context.Context, duration time.Duration) (string, map[string][]TimeSeriesPoint, error) {