Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Keep native host I/O visible beside stored CPU history

pulse-triage[bot]495c0cc6ab19
diff --git a/internal/api/router.go b/internal/api/router.go
index 424602b758..3338fbfc9d 100644
--- a/internal/api/router.go
+++ b/internal/api/router.go
@@ -6816,7 +6816,7 @@ func (r *Router) handleMetricsHistory(w http.ResponseWriter, req *http.Request)
return points
}
- guestChartMetrics := func() (map[string][]monitoring.MetricPoint, bool) {
+ guestChartMetrics := func(metricTypes ...string) (map[string][]monitoring.MetricPoint, bool) {
inMemoryKey := resourceID
sqlResourceType := ""
switch runtimeResourceType {
@@ -6838,7 +6838,7 @@ func (r *Router) handleMetricsHistory(w http.ResponseWriter, req *http.Request)
default:
return nil, false
}
- return monitor.GetGuestMetricsForChart(inMemoryKey, sqlResourceType, resourceID, duration), true
+ return monitor.GetGuestMetricsForChart(inMemoryKey, sqlResourceType, resourceID, duration, metricTypes...), true
}
guestChartSource := func() string {
if mock.IsMockEnabled() {
@@ -6893,7 +6893,7 @@ func (r *Router) handleMetricsHistory(w http.ResponseWriter, req *http.Request)
switch runtimeResourceType {
case "vm", "system-container", "oci-container":
- metrics, _ := guestChartMetrics()
+ metrics, _ := guestChartMetrics(metricType)
points := metrics[metricType]
if len(points) == 0 {
livePoints := liveMetricPoints(runtimeResourceType, resourceID)
@@ -6904,7 +6904,7 @@ func (r *Router) handleMetricsHistory(w http.ResponseWriter, req *http.Request)
}
return buildHistoryPoints(points, stepSecs), guestChartSource(), true
case "docker-host":
- metrics, _ := guestChartMetrics()
+ metrics, _ := guestChartMetrics(metricType)
points := metrics[metricType]
if len(points) == 0 {
livePoints := liveMetricPoints(runtimeResourceType, resourceID)
@@ -6915,7 +6915,7 @@ func (r *Router) handleMetricsHistory(w http.ResponseWriter, req *http.Request)
}
return buildHistoryPoints(points, stepSecs), guestChartSource(), true
case "agent":
- metrics, _ := guestChartMetrics()
+ metrics, _ := guestChartMetrics(metricType)
points := metrics[metricType]
if len(points) == 0 {
points = monitor.GetNodeMetrics(resourceID, metricType, duration)
@@ -6929,7 +6929,7 @@ func (r *Router) handleMetricsHistory(w http.ResponseWriter, req *http.Request)
}
return buildHistoryPoints(points, stepSecs), guestChartSource(), true
case "app-container":
- metrics, _ := guestChartMetrics()
+ metrics, _ := guestChartMetrics(metricType)
points := metrics[metricType]
if len(points) == 0 {
livePoints := liveMetricPoints(runtimeResourceType, resourceID)
@@ -6940,7 +6940,7 @@ func (r *Router) handleMetricsHistory(w http.ResponseWriter, req *http.Request)
}
return buildHistoryPoints(points, stepSecs), guestChartSource(), true
case "k8s":
- metrics, _ := guestChartMetrics()
+ metrics, _ := guestChartMetrics(metricType)
points := metrics[metricType]
if len(points) == 0 {
livePoints := liveMetricPoints(runtimeResourceType, resourceID)
@@ -7404,6 +7404,33 @@ func (r *Router) handleMetricsHistory(w http.ResponseWriter, req *http.Request)
}
}
+ // A stored CPU/capacity row does not establish host I/O coverage.
+ // Ask only for missing directions; retain every stored series and
+ // the native parser's complete-member/coincident-time requirement.
+ if runtimeResourceType == "agent" && !mock.IsMockEnabled() {
+ var missing []string
+ for _, metric := range []string{"diskread", "diskwrite"} {
+ if len(apiData[metric]) == 0 {
+ missing = append(missing, metric)
+ }
+ }
+ if len(missing) > 0 {
+ fallback, _ := guestChartMetrics(missing...)
+ for _, metric := range missing {
+ var withinWindow []monitoring.MetricPoint
+ for _, point := range fallback[metric] {
+ if !point.Timestamp.Before(start) && !point.Timestamp.After(end) {
+ withinWindow = append(withinWindow, point)
+ }
+ }
+ if len(withinWindow) > 0 {
+ apiData[metric] = buildHistoryPoints(withinWindow, stepSecs)
+ source = historySourceMemory
+ }
+ }
+ }
+ }
+
// QueryAll can return a non-empty but incomplete mock metric map. Fill
// only the absent series from the deterministic chart fallback so one
// populated metric cannot suppress entire drawer groups.
diff --git a/internal/api/truenas_host_io_history_test.go b/internal/api/truenas_host_io_history_test.go
new file mode 100644
index 0000000000..29950fd786
--- /dev/null
+++ b/internal/api/truenas_host_io_history_test.go
@@ -0,0 +1,303 @@
+package api
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/config"
+ "github.com/rcourtman/pulse-go-rewrite/internal/models"
+ "github.com/rcourtman/pulse-go-rewrite/internal/monitoring"
+ "github.com/rcourtman/pulse-go-rewrite/internal/truenas"
+ "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
+ "github.com/rcourtman/pulse-go-rewrite/pkg/metrics"
+)
+
+// This fixture models CORE's REST graph contract, not a reporter's appliance.
+// Live disjoint disk samples survive individually but cannot form a host sum.
+// History phases independently control whether a complete native sum exists.
+type hostIORESTFixture struct {
+ server *httptest.Server
+ phase atomic.Int32
+ reads atomic.Int32
+ end atomic.Int64
+}
+
+func newHostIORESTFixture(t *testing.T) *hostIORESTFixture {
+ t.Helper()
+ f := &hostIORESTFixture{}
+ f.server = httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ switch r.URL.Path {
+ case "/api/v2.0/system/info":
+ fmt.Fprint(w, `{"hostname":"same-core-name","version":"TrueNAS-13.0-U6.8","cores":8,"physmem":17179869184}`)
+ case "/api/v2.0/pool":
+ fmt.Fprint(w, `[{"id":1,"name":"tank","status":"ONLINE","size":1000,"allocated":400,"free":600}]`)
+ case "/api/v2.0/disk":
+ fmt.Fprint(w, `[{"identifier":"disk-a","name":"da0","serial":"host-io-a","type":"HDD","size":1000},{"identifier":"disk-b","name":"da1","serial":"host-io-b","type":"HDD","size":1000}]`)
+ case "/api/v2.0/pool/dataset", "/api/v2.0/alert/list":
+ fmt.Fprint(w, `[]`)
+ case "/api/v2.0/reporting/graphs":
+ if f.phase.Load() == 2 {
+ fmt.Fprint(w, `[{"name":"disk","identifiers":["da0","da1","da2"]}]`)
+ } else {
+ fmt.Fprint(w, `[{"name":"disk","identifiers":["da0","da1"]}]`)
+ }
+ case "/api/v2.0/reporting/get_data":
+ f.reads.Add(1)
+ var request struct {
+ Graphs []struct{ Name, Identifier string }
+ Query struct{ Start, End int64 } `json:"reporting_query"`
+ }
+ if r.Method != http.MethodPost || json.NewDecoder(r.Body).Decode(&request) != nil || request.Query.End <= request.Query.Start {
+ http.Error(w, "invalid synthetic reporting request", 400)
+ return
+ }
+ f.end.Store(request.Query.End)
+ var responses []map[string]any
+ for _, graph := range request.Graphs {
+ start, step := request.Query.End-3540, int64(1160)
+ var legend []string
+ var row []any
+ switch graph.Name {
+ case "cpu":
+ legend, row = []string{"usage"}, []any{12.}
+ case "disk":
+ legend, row = []string{"disk_octets_read", "disk_octets_write"}, []any{100., 200.}
+ if graph.Identifier == "da1" {
+ row = []any{1000., 2000.}
+ }
+ if f.phase.Load() == 3 {
+ row = []any{0., 0.}
+ }
+ if f.phase.Load() == 4 && graph.Identifier == "da1" {
+ row[0] = nil
+ }
+ default:
+ continue
+ }
+ rows := []any{row, row, row, row}
+ if graph.Identifier == "da2" {
+ rows = nil
+ }
+ if f.phase.Load() == 0 && graph.Name == "disk" {
+ start, step, rows = request.Query.End-20, 10, []any{row, row}
+ if graph.Identifier == "da1" {
+ start += 5
+ }
+ }
+ responses = append(responses, map[string]any{"name": graph.Name, "identifier": graph.Identifier, "legend": legend, "start": start, "end": request.Query.End, "step": step, "data": rows})
+ }
+ if err := json.NewEncoder(w).Encode(responses); err != nil {
+ t.Error(err)
+ }
+ default:
+ http.NotFound(w, r)
+ }
+ }))
+ t.Cleanup(f.server.Close)
+ return f
+}
+
+func (f *hostIORESTFixture) provider(t *testing.T, connection string) *truenas.Provider {
+ t.Helper()
+ client, err := truenas.NewClient(truenas.ClientConfig{Host: f.server.URL, APIKey: "synthetic-host-io-key", InsecureSkipVerify: true, Fingerprint: fmt.Sprintf("%x", sha256.Sum256(f.server.Certificate().Raw))})
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(client.Close)
+ p := truenas.NewLiveProviderForConnection(&truenas.APIFetcher{Client: client}, connection)
+ if err := p.Refresh(context.Background()); err != nil {
+ t.Fatal(err)
+ }
+ return p
+}
+
+type hostIOHistoryPoint struct {
+ Timestamp int64 `json:"timestamp"`
+ Value float64 `json:"value"`
+ Min float64 `json:"min"`
+ Max float64 `json:"max"`
+}
+
+// Exercise native CORE collection -> canonical ingestion -> persisted HTTP,
+// including the drawer's all-metric query with unrelated covered CPU history.
+func TestTrueNASHostIOHistoryWithPartialStore(t *testing.T) {
+ previous := truenas.IsFeatureEnabled()
+ truenas.SetFeatureEnabled(true)
+ t.Cleanup(func() { truenas.SetFeatureEnabled(previous) })
+ previousMulti := IsMultiTenantEnabled()
+ SetMultiTenantEnabled(true)
+ t.Cleanup(func() { SetMultiTenantEnabled(previousMulti) })
+ t.Setenv("PULSE_DEV", "true")
+ f, other := newHostIORESTFixture(t), newHostIORESTFixture(t)
+ provider, otherProvider := f.provider(t, "host-io-target"), other.provider(t, "other-target")
+ snapshot := provider.Snapshot()
+ if snapshot.System.Telemetry.DiskRead || snapshot.System.Telemetry.DiskWrite || len(snapshot.System.DiskReadRates) != 2 {
+ t.Fatal("fixture did not establish complete per-disk readings with an absent host total")
+ }
+ store, err := metrics.NewStore(metrics.DefaultConfig(t.TempDir()))
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { _ = store.Close() })
+ poller := monitoring.NewTrueNASPoller(nil, time.Minute, nil)
+ setUnexportedField(t, poller, "providersByOrg", map[string]map[string]*truenas.Provider{"org-host-io": {"host-io-target": provider, "other-target": otherProvider}})
+ setUnexportedField(t, poller, "cachedRecordsByOrg", map[string]map[string][]unifiedresources.IngestRecord{"org-host-io": {"host-io-target": provider.Records()}})
+ monitor := &monitoring.Monitor{}
+ monitor.SetOrgID("org-host-io")
+ setUnexportedField(t, monitor, "state", models.NewState())
+ ring := monitoring.NewMetricsHistory(1024, 24*time.Hour)
+ setUnexportedField(t, monitor, "metricsHistory", ring)
+ setUnexportedField(t, monitor, "metricsStore", store)
+ monitor.SetResourceStoreAndProviders(unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil)), map[unifiedresources.DataSource]monitoring.MonitorSupplementalRecordsProvider{unifiedresources.SourceTrueNAS: poller})
+ store.Flush()
+ now := time.Now().UTC().Truncate(time.Second)
+ // Simulate an established CPU ring. That must not certify an absent I/O series.
+ for _, offset := range []time.Duration{-55 * time.Minute, -30 * time.Minute, -time.Minute} {
+ ring.AddGuestMetric("agent:host-io-target", "cpu", 19, now.Add(offset))
+ }
+ const token = "host-io-reader.1234567890123456789"
+ readToken := newTokenRecord(t, token, []string{config.ScopeMonitoringRead}, nil)
+ readToken.OrgID = "org-host-io"
+ const wrongScope = "host-io-settings.1234567890123456789"
+ scopeToken := newTokenRecord(t, wrongScope, []string{config.ScopeSettingsRead}, nil)
+ scopeToken.OrgID = "org-host-io"
+ cfg := newTestConfigWithTokens(t, readToken, scopeToken)
+ cfg.AuthUser, cfg.AuthPass = "fixture-admin", "synthetic-unused-password-hash"
+ for _, org := range []string{"org-host-io", "org-other"} {
+ if err := os.MkdirAll(filepath.Join(cfg.DataPath, "orgs", org), 0700); err != nil {
+ t.Fatal(err)
+ }
+ }
+ router := NewRouter(cfg, monitor, nil, nil, nil, "test")
+ t.Cleanup(router.ShutdownBackgroundWorkers)
+ t.Cleanup(router.ShutdownResourceStores)
+ t.Cleanup(router.ShutdownRBAC)
+ mtm := monitoring.NewMultiTenantMonitor(cfg, nil, nil)
+ setUnexportedField(t, mtm, "monitors", map[string]*monitoring.Monitor{"org-host-io": monitor, "org-other": {}})
+ router.mtMonitor = mtm
+ router.resourceHandlers.SetTenantStateProvider(NewMultiTenantStateProvider(mtm, monitor))
+ router.setMonitorSupplementalRecordsProvider(unifiedresources.SourceTrueNAS, poller)
+ request := func(raw, org, resourceType, id, metric string) *httptest.ResponseRecorder {
+ req := httptest.NewRequest(http.MethodGet, "/api/metrics-store/history?resourceType="+resourceType+"&resourceId="+id+"&metric="+metric+"&range=1h&maxPoints=240", nil)
+ req.Header.Set("X-Pulse-Org-ID", org)
+ if raw != "" {
+ req.Header.Set("X-API-Token", raw)
+ }
+ rec := httptest.NewRecorder()
+ router.Handler().ServeHTTP(rec, req)
+ return rec
+ }
+ f.reads.Store(0)
+ other.reads.Store(0)
+ for _, test := range []struct {
+ raw, org string
+ want int
+ }{{"", "org-host-io", 401}, {wrongScope, "org-host-io", 403}, {token, "org-other", 403}} {
+ if rec := request(test.raw, test.org, "agent", "host-io-target", ""); rec.Code != test.want || f.reads.Load() != 0 || other.reads.Load() != 0 {
+ t.Fatalf("auth/scope/tenant gate: status %d want %d reads %d/%d", rec.Code, test.want, f.reads.Load(), other.reads.Load())
+ }
+ }
+ // Actual per-disk persistence is independent of the native host fallback.
+ for _, serial := range []string{"host-io-a", "host-io-b"} {
+ rec := request(token, "org-host-io", "disk", serial, "diskread")
+ var result struct{ Points []hostIOHistoryPoint }
+ if rec.Code != 200 || json.Unmarshal(rec.Body.Bytes(), &result) != nil || len(result.Points) == 0 {
+ t.Fatalf("individual disk lost persisted I/O: %d %s", rec.Code, rec.Body.String())
+ }
+ }
+ if f.reads.Load() != 0 || other.reads.Load() != 0 {
+ t.Fatal("persisted per-disk I/O queried native host History")
+ }
+ var storedCPU []hostIOHistoryPoint
+ for _, test := range []struct {
+ name string
+ phase int32
+ read, write bool
+ value float64
+ }{{"complete", 1, true, true, 1100}, {"empty-member", 2, false, false, 0}, {"measured-zero", 3, true, true, 0}, {"write-only", 4, false, true, 0}, {"disjoint", 0, false, false, 0}} {
+ t.Run(test.name, func(t *testing.T) {
+ f.phase.Store(test.phase)
+ before := f.reads.Load()
+ rec := request(token, "org-host-io", "agent", "host-io-target", "")
+ var result struct {
+ Metrics map[string][]hostIOHistoryPoint
+ }
+ if rec.Code != 200 || json.Unmarshal(rec.Body.Bytes(), &result) != nil {
+ t.Fatalf("host history %d %s", rec.Code, rec.Body.String())
+ }
+ if (len(result.Metrics["diskread"]) > 0) != test.read || (len(result.Metrics["diskwrite"]) > 0) != test.write {
+ t.Fatalf("host I/O completeness: read=%d write=%d want %t/%t", len(result.Metrics["diskread"]), len(result.Metrics["diskwrite"]), test.read, test.write)
+ }
+ if f.reads.Load()-before != 1 || other.reads.Load() != 0 {
+ t.Fatalf("host fallback read other appliance or lost its one-request bound: reads %d/%d", f.reads.Load()-before, other.reads.Load())
+ }
+ if len(result.Metrics["cpu"]) != 1 || result.Metrics["cpu"][0].Value != 12 {
+ t.Fatalf("host fallback replaced stored CPU: %+v", result.Metrics["cpu"])
+ }
+ if storedCPU == nil {
+ storedCPU = result.Metrics["cpu"]
+ } else if result.Metrics["cpu"][0] != storedCPU[0] {
+ t.Fatal("native supplement changed stored CPU time/bounds")
+ }
+ for _, metric := range []string{"diskread", "diskwrite"} {
+ for i, point := range result.Metrics[metric] {
+ value := test.value
+ if metric == "diskwrite" && test.phase != 3 {
+ value = 2200
+ }
+ ts := f.end.Load() - 3540 + int64(i)*1160
+ if point.Value != value || point.Min != value || point.Max != value || point.Timestamp != (ts/15*15+7)*1000 {
+ t.Fatalf("native rate/time/bucket/bounds changed: %s %+v", metric, point)
+ }
+ }
+ }
+ t.Logf("HOST_IO_BROWSER %s %s", test.name, rec.Body.String())
+ })
+ }
+ // Single-series reads must qualify that series, not a covered CPU sibling.
+ f.phase.Store(1)
+ before := f.reads.Load()
+ rec := request(token, "org-host-io", "agent", "host-io-target", "diskread")
+ var single struct{ Points []hostIOHistoryPoint }
+ if rec.Code != 200 || json.Unmarshal(rec.Body.Bytes(), &single) != nil || len(single.Points) != 4 || f.reads.Load()-before != 1 || other.reads.Load() != 0 {
+ t.Fatalf("single host I/O hidden by covered CPU: status %d points %d reads %d/%d", rec.Code, len(single.Points), f.reads.Load()-before, other.reads.Load())
+ }
+ // Existing stored directions are not replaced by a larger native series.
+ ioTime := now.Add(-20 * time.Second)
+ store.WriteBatchSync([]metrics.WriteMetric{{ResourceType: "agent", ResourceID: "host-io-target", MetricType: "diskread", Value: 777, Timestamp: ioTime, Tier: metrics.TierRaw}})
+ rec = request(token, "org-host-io", "agent", "host-io-target", "")
+ var partial struct {
+ Metrics map[string][]hostIOHistoryPoint
+ }
+ if rec.Code != 200 || json.Unmarshal(rec.Body.Bytes(), &partial) != nil || len(partial.Metrics["diskread"]) != 1 || partial.Metrics["diskread"][0].Value != 777 || len(partial.Metrics["diskwrite"]) != 4 {
+ t.Fatalf("native fill replaced stored direction: %s", rec.Body.String())
+ }
+ store.WriteBatchSync([]metrics.WriteMetric{{ResourceType: "agent", ResourceID: "host-io-target", MetricType: "diskwrite", Value: 888, Timestamp: ioTime, Tier: metrics.TierRaw}})
+ before = f.reads.Load()
+ rec = request(token, "org-host-io", "agent", "host-io-target", "")
+ if rec.Code != 200 || f.reads.Load() != before || other.reads.Load() != 0 {
+ t.Fatal("complete stored I/O needlessly queried native History")
+ }
+ if err := store.Close(); err != nil {
+ t.Fatal(err)
+ }
+ if rec := request(token, "org-host-io", "agent", "host-io-target", ""); rec.Code != 500 || f.reads.Load() != before {
+ t.Fatal("failed store read became successful native history")
+ }
+ if cfg.RemoveAPIToken(readToken.ID) == nil {
+ t.Fatal("missing fixture token")
+ }
+ if rec := request(token, "org-host-io", "agent", "host-io-target", ""); rec.Code != 401 || f.reads.Load() != before {
+ t.Fatal("withdrawn reader reached native history")
+ }
+}
diff --git a/internal/monitoring/monitor_metrics.go b/internal/monitoring/monitor_metrics.go
index e52b8e9c34..03d80c145b 100644
--- a/internal/monitoring/monitor_metrics.go
+++ b/internal/monitoring/monitor_metrics.go
@@ -72,6 +72,12 @@ type MonitorGuestMetricHistoryProvider interface {
GuestMetricHistory(m *Monitor, orgID string, resourceType string, duration time.Duration) map[string]map[string][]MetricPoint
}
+// MonitorTargetedGuestMetricHistoryProvider avoids querying other appliances
+// when a single resource needs a source-native chart fallback.
+type MonitorTargetedGuestMetricHistoryProvider interface {
+ GuestMetricHistoryForResource(m *Monitor, orgID, resourceType, resourceID string, duration time.Duration) map[string][]MetricPoint
+}
+
// GetGuestMetrics returns historical metrics for a guest
func (m *Monitor) GetGuestMetrics(guestID string, duration time.Duration) map[string][]MetricPoint {
if m == nil || m.metricsHistory == nil {
@@ -106,17 +112,18 @@ func (m *Monitor) GetGuestMetrics(guestID string, duration time.Duration) map[st
// inMemoryKey is the key used in the in-memory buffer (e.g. "docker:abc123").
// sqlResourceType/sqlResourceID are the type/id used in the SQLite store
// (e.g. "dockerContainer"/"abc123").
-func (m *Monitor) GetGuestMetricsForChart(inMemoryKey, sqlResourceType, sqlResourceID string, duration time.Duration) map[string][]MetricPoint {
+func (m *Monitor) GetGuestMetricsForChart(inMemoryKey, sqlResourceType, sqlResourceID string, duration time.Duration, metricTypes ...string) map[string][]MetricPoint {
+ requestedMetricTypes := normalizeChartMetricTypes(metricTypes)
if mock.IsMockEnabled() {
- return m.mockGuestMetricsForChart(inMemoryKey, sqlResourceType, sqlResourceID, duration, m.GetGuestMetrics(inMemoryKey, duration))
+ return filterChartMetricMap(m.mockGuestMetricsForChart(inMemoryKey, sqlResourceType, sqlResourceID, duration, m.GetGuestMetrics(inMemoryKey, duration)), requestedMetricTypes)
}
inMemoryResult := map[string][]MetricPoint{}
- if m.shouldLoadGuestChartInMemory(inMemoryKey, nil, duration) {
- inMemoryResult = m.GetGuestMetrics(inMemoryKey, duration)
+ if m.shouldLoadGuestChartInMemory(inMemoryKey, requestedMetricTypes, duration) {
+ inMemoryResult = filterChartMetricMap(m.GetGuestMetrics(inMemoryKey, duration), requestedMetricTypes)
}
- if hasSufficientChartMapCoverage(inMemoryResult, duration) {
+ if hasSufficientChartMapCoverageForMetrics(inMemoryResult, duration, requestedMetricTypes) {
return inMemoryResult
}
@@ -125,12 +132,15 @@ func (m *Monitor) GetGuestMetricsForChart(inMemoryKey, sqlResourceType, sqlResou
best := inMemoryResult
if m.metricsStore != nil {
if converted, ok := m.queryStoreMetricMapWithGapFillAliases(sqlResourceType, sqlResourceID, duration); ok {
- best = mergeGuestMetricHistory(best, converted, duration)
+ best = mergeGuestMetricHistory(best, filterChartMetricMap(converted, requestedMetricTypes), duration)
}
}
- if nativeHistory := m.nativeGuestMetricHistory(sqlResourceType, duration); len(nativeHistory) > 0 {
+ if len(requestedMetricTypes) > 0 && hasSufficientChartMapCoverageForMetrics(best, duration, requestedMetricTypes) {
+ return best
+ }
+ if nativeHistory := m.nativeGuestMetricHistory(sqlResourceType, duration, sqlResourceID); len(nativeHistory) > 0 {
if nativeMetrics, ok := nativeHistory[sqlResourceID]; ok {
- best = mergeGuestMetricHistory(best, nativeMetrics, duration)
+ best = mergeGuestMetricHistory(best, filterChartMetricMap(nativeMetrics, requestedMetricTypes), duration)
}
}
return best
@@ -806,7 +816,7 @@ func (m *Monitor) GetStorageMetricsForChartBatch(
return result
}
-func (m *Monitor) nativeGuestMetricHistory(resourceType string, duration time.Duration) map[string]map[string][]MetricPoint {
+func (m *Monitor) nativeGuestMetricHistory(resourceType string, duration time.Duration, resourceIDs ...string) map[string]map[string][]MetricPoint {
providers := m.supplementalProviderSnapshot()
if len(providers) == 0 {
return nil
@@ -821,6 +831,14 @@ func (m *Monitor) nativeGuestMetricHistory(resourceType string, duration time.Du
history := make(map[string]map[string][]MetricPoint)
for _, provider := range providers {
+ if targeted, ok := provider.(MonitorTargetedGuestMetricHistoryProvider); ok && len(resourceIDs) == 1 {
+ id := resourceIDs[0]
+ metricMap := targeted.GuestMetricHistoryForResource(m, orgID, resourceType, id, duration)
+ if len(metricMap) > 0 {
+ history[id] = mergeGuestMetricHistory(history[id], metricMap, duration)
+ }
+ continue
+ }
historyProvider, ok := provider.(MonitorGuestMetricHistoryProvider)
if !ok {
continue
diff --git a/internal/monitoring/monitor_metrics_requested_history_test.go b/internal/monitoring/monitor_metrics_requested_history_test.go
new file mode 100644
index 0000000000..703d75d5ba
--- /dev/null
+++ b/internal/monitoring/monitor_metrics_requested_history_test.go
@@ -0,0 +1,57 @@
+package monitoring
+
+import (
+ "testing"
+ "time"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
+)
+
+type targetedChartHistoryFixture struct {
+ stubGuestMetricHistoryProvider
+ reads, bulkReads int
+ id, org, kind string
+}
+
+func (s *targetedChartHistoryFixture) GuestMetricHistory(*Monitor, string, string, time.Duration) map[string]map[string][]MetricPoint {
+ s.bulkReads++
+ return nil
+}
+
+func (s *targetedChartHistoryFixture) GuestMetricHistoryForResource(_ *Monitor, org, kind, id string, _ time.Duration) map[string][]MetricPoint {
+ s.reads++
+ s.id, s.org, s.kind = id, org, kind
+ return s.stubGuestMetricHistoryProvider.GuestMetricHistory(nil, org, kind, time.Hour)[id]
+}
+
+func TestGuestChartRequestedCoverageIsPerSeries(t *testing.T) {
+ t.Parallel()
+ for _, duration := range []time.Duration{time.Hour, 24 * time.Hour} {
+ t.Run(duration.String(), func(t *testing.T) {
+ monitor := newChartFallbackTestMonitor(t)
+ monitor.SetOrgID("org-targeted")
+ now := time.Now().UTC().Truncate(time.Second)
+ points := []MetricPoint{{Timestamp: now.Add(-duration + time.Minute), Value: 100}, {Timestamp: now.Add(-time.Minute), Value: 0}}
+ provider := &targetedChartHistoryFixture{stubGuestMetricHistoryProvider: stubGuestMetricHistoryProvider{history: map[string]map[string][]MetricPoint{
+ "host-target": {"diskread": points, "diskwrite": points, "cpu": {{Timestamp: now, Value: 999}}},
+ }}}
+ monitor.supplementalProviders = map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider{unifiedresources.SourceTrueNAS: provider}
+ for _, point := range points {
+ monitor.metricsHistory.AddGuestMetric("agent:host-target", "cpu", 12, point.Timestamp)
+ }
+ result := monitor.GetGuestMetricsForChart("agent:host-target", "agent", "host-target", duration, "diskread", "diskwrite")
+ if len(result["diskread"]) != 2 || len(result["diskwrite"]) != 2 || result["diskread"][1].Value != 0 || len(result["cpu"]) != 0 || provider.reads != 1 || provider.bulkReads != 0 {
+ t.Fatalf("covered CPU hid requested IO or target escaped: %+v reads %d/%d", result, provider.reads, provider.bulkReads)
+ }
+ if provider.id != "host-target" || provider.org != "org-targeted" || provider.kind != "agent" {
+ t.Fatalf("target binding changed: %+v", provider)
+ }
+ writeRawMetricBatch(t, monitor.metricsStore, "agent", "host-target", "diskread", points)
+ writeRawMetricBatch(t, monitor.metricsStore, "agent", "host-target", "diskwrite", points)
+ monitor.GetGuestMetricsForChart("agent:host-target", "agent", "host-target", duration, "diskread", "diskwrite")
+ if provider.reads != 1 {
+ t.Fatal("covered requested store series queried native provider")
+ }
+ })
+ }
+}
diff --git a/internal/monitoring/truenas_host_history_target_test.go b/internal/monitoring/truenas_host_history_target_test.go
new file mode 100644
index 0000000000..1ffef8c7e2
--- /dev/null
+++ b/internal/monitoring/truenas_host_history_target_test.go
@@ -0,0 +1,82 @@
+package monitoring
+
+import (
+ "context"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/truenas"
+)
+
+type targetedTrueNASHistoryFetcher struct {
+ controllableTrueNASHistoryFetcher
+ reads atomic.Int32
+ entered chan struct{}
+ release chan struct{}
+}
+
+func (f *targetedTrueNASHistoryFetcher) SystemMetricHistory(ctx context.Context, duration time.Duration) (*truenas.SystemMetricHistory, error) {
+ f.reads.Add(1)
+ if f.entered != nil {
+ close(f.entered)
+ <-f.release
+ }
+ return f.controllableTrueNASHistoryFetcher.SystemMetricHistory(ctx, duration)
+}
+
+func TestTrueNASHostHistoryTargetAndLifetime(t *testing.T) {
+ previous := truenas.IsFeatureEnabled()
+ truenas.SetFeatureEnabled(true)
+ t.Cleanup(func() { truenas.SetFeatureEnabled(previous) })
+ newProvider := func(connection string) (*truenas.Provider, *targetedTrueNASHistoryFetcher) {
+ fixture := truenas.DefaultFixtures()
+ fixture.System.Hostname = "same-name"
+ f := &targetedTrueNASHistoryFetcher{controllableTrueNASHistoryFetcher: controllableTrueNASHistoryFetcher{snapshot: &fixture, systemHistory: &truenas.SystemMetricHistory{DiskReadRate: []truenas.TimeSeriesPoint{{Timestamp: time.Now().Add(-time.Minute), Value: 0}}}}}
+ p := truenas.NewLiveProviderForConnection(f, connection)
+ if err := p.Refresh(context.Background()); err != nil {
+ t.Fatal(err)
+ }
+ return p, f
+ }
+ target, targetFetcher := newProvider("target")
+ other, otherFetcher := newProvider("other")
+ poller := NewTrueNASPoller(nil, time.Minute, nil)
+ poller.providersByOrg["org-a"] = map[string]*truenas.Provider{"target": target, "other": other}
+ for _, test := range []struct{ org, kind, id string }{{"org-b", "agent", "target"}, {"org-a", "disk", "target"}, {"org-a", "agent", "missing"}, {"org-a", "agent", "same-name"}, {"org-a", "agent", ""}} {
+ if result := poller.GuestMetricHistoryForResource(nil, test.org, test.kind, test.id, time.Hour); len(result) != 0 || targetFetcher.reads.Load() != 0 || otherFetcher.reads.Load() != 0 {
+ t.Fatalf("nonmatching target reached native history: %+v", test)
+ }
+ }
+ result := poller.GuestMetricHistoryForResource(nil, "org-a", "agent", "target", time.Hour)
+ if len(result["diskread"]) != 1 || result["diskread"][0].Value != 0 || targetFetcher.reads.Load() != 1 || otherFetcher.reads.Load() != 0 {
+ t.Fatalf("exact target/zero lost or other same-name host read: %+v", result)
+ }
+ for _, action := range []string{"remove", "replace", "disable"} {
+ t.Run(action, func(t *testing.T) {
+ p, f := newProvider("target")
+ f.entered, f.release = make(chan struct{}), make(chan struct{})
+ poller.mu.Lock()
+ poller.providersByOrg["org-a"]["target"] = p
+ poller.mu.Unlock()
+ truenas.SetFeatureEnabled(true)
+ done := make(chan map[string][]MetricPoint, 1)
+ go func() { done <- poller.GuestMetricHistoryForResource(nil, "org-a", "agent", "target", time.Hour) }()
+ <-f.entered
+ poller.mu.Lock()
+ switch action {
+ case "remove":
+ delete(poller.providersByOrg["org-a"], "target")
+ case "replace":
+ poller.providersByOrg["org-a"]["target"] = target
+ case "disable":
+ truenas.SetFeatureEnabled(false)
+ }
+ poller.mu.Unlock()
+ close(f.release)
+ if result := <-done; len(result) != 0 {
+ t.Fatalf("withdrawn provider returned native history: %+v", result)
+ }
+ })
+ }
+}
diff --git a/internal/monitoring/truenas_poller.go b/internal/monitoring/truenas_poller.go
index c620be45ff..8ad2b55f40 100644
--- a/internal/monitoring/truenas_poller.go
+++ b/internal/monitoring/truenas_poller.go
@@ -1075,6 +1075,20 @@ func (p *TrueNASPoller) PhysicalDiskTemperatureHistory(_ *Monitor, orgID string,
// GuestMetricHistory exposes native TrueNAS host history through the canonical
// guest-chart boundary when local Pulse history is shallow.
func (p *TrueNASPoller) GuestMetricHistory(_ *Monitor, orgID string, resourceType string, duration time.Duration) map[string]map[string][]MetricPoint {
+ return p.guestMetricHistory(orgID, resourceType, "", duration)
+}
+
+// GuestMetricHistoryForResource selects the exact tenant-local host before any
+// native read. A missing host must not turn a drawer request into a fleet scan.
+func (p *TrueNASPoller) GuestMetricHistoryForResource(_ *Monitor, orgID, resourceType, resourceID string, duration time.Duration) map[string][]MetricPoint {
+ resourceID = strings.TrimSpace(resourceID)
+ if resourceID == "" {
+ return nil
+ }
+ return p.guestMetricHistory(orgID, resourceType, resourceID, duration)[resourceID]
+}
+
+func (p *TrueNASPoller) guestMetricHistory(orgID, resourceType, selectedID string, duration time.Duration) map[string]map[string][]MetricPoint {
if p == nil || !truenas.IsFeatureEnabled() || strings.TrimSpace(resourceType) != "agent" {
return nil
}
@@ -1094,6 +1108,9 @@ func (p *TrueNASPoller) GuestMetricHistory(_ *Monitor, orgID string, resourceTyp
history := make(map[string]map[string][]MetricPoint)
for _, entry := range entries {
+ if selectedID != "" && entry.provider.SystemMetricResourceID() != selectedID {
+ continue
+ }
resourceID, nativeHistory, err := entry.provider.SystemMetricHistory(ctx, duration)
if err != nil {
log.Warn().
@@ -1107,7 +1124,14 @@ func (p *TrueNASPoller) GuestMetricHistory(_ *Monitor, orgID string, resourceTyp
continue
}
resourceID = strings.TrimSpace(resourceID)
- if resourceID == "" || len(nativeHistory) == 0 {
+ if resourceID == "" || len(nativeHistory) == 0 || (selectedID != "" && resourceID != selectedID) {
+ continue
+ }
+ // Do not return a provider removed or replaced while its read was in flight.
+ p.mu.Lock()
+ current := truenas.IsFeatureEnabled() && p.providersByOrg[orgID][entry.connectionID] == entry.provider
+ p.mu.Unlock()
+ if !current {
continue
}
diff --git a/internal/truenas/provider.go b/internal/truenas/provider.go
index 991534bfcf..9f1cfa268e 100644
--- a/internal/truenas/provider.go
+++ b/internal/truenas/provider.go
@@ -324,6 +324,20 @@ func (p *Provider) GetAppConfig(_ context.Context, appID string) (*AppConfigResu
return result, nil
}
+// SystemMetricResourceID reports the same native host identity as Records and
+// SystemMetricHistory, without starting a provider request.
+func (p *Provider) SystemMetricResourceID() string {
+ if p == nil {
+ return ""
+ }
+ p.mu.Lock()
+ defer p.mu.Unlock()
+ if p.lastSnapshot == nil {
+ return ""
+ }
+ return trueNASSystemMetricResourceID(p.connectionID, p.lastSnapshot.System)
+}
+
// SystemMetricHistory returns canonical host-chart metrics for the TrueNAS
// appliance keyed by the shared agent metrics resource ID.
func (p *Provider) SystemMetricHistory(ctx context.Context, duration time.Duration) (string, map[string][]TimeSeriesPoint, error) {