Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Restore CORE reporting and native History on current main
pulse-triage[bot]4771e3ace1cb
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex a89ad201c2..ba0ab40318 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -2006,8 +2006,8 @@ are not native appliance or containing-release acceptance.A connection proven to run a recognized legacy CORE/FreeNAS release has no`reporting.realtime` JSON-RPC subscription, so live system telemetry andhost-chart history are reconstructed from the REST reporting API-(`reporting.get_data`, served as a POST endpoint keyed by its `graphs` and-`query` parameters). The reporting `memory` and `arcsize` graphs supply the+(`reporting.get_data`, served as a POST endpoint keyed by the middleware+schema names `graphs` and `reporting_query`). The reporting `memory` and `arcsize` graphs supply theavailable and ARC readings that the JSON-RPC realtime subscription otherwiseprovides. When no available-memory reading can be established, the providermust not derive usage from an absent available value: the system memory metric is@@ -2024,6 +2024,30 @@ confirmation.+### CORE REST envelope and captured History targets — issues #2077 and #2964++Legacy REST `POST /reporting/get_data` sends exactly `graphs` and+`reporting_query`. WebSocket's positional second argument is not a REST key+named `query`: CORE rejects that unknown attribute before collecting any graph.+Live reads and native History share this envelope and unchanged aggregate,+start/end range, graph catalogue and optional-graph isolation. TLS, authentication,+transport selection and per-metric zero/unknown semantics are unchanged.++The monitor's resource snapshot carries its captured generation's resolved+History target into the detached REST seed, including authoritative absence.+It never guesses a native connection ID from a display name, consults a later+registry generation or decorates shared view rows. This restores the existing+canonical History identity contract rather than adding a route or schema.++`TestCORE13RESTReportingQueryResourceJourney` and+`TestCORE13RESTReportingChartJourney` exercise the real pinned-TLS+client/provider, authenticated resources and chart route, native History,+free/ARC memory, unavailable/partial/zero/recovered readings, retained historical+observations and access revocation. `TestUnifiedSnapshotPreservesCapturedHistoryTargets`+protects captured coordinates, authoritative absence and detached ownership.+These schema-faithful synthetic controls are not native appliance acceptance,+reporter confirmation, data-pool recovery or release availability.+### TrueNAS partial samples and nonblocking telemetry diagnostics — issue #2077Live system identity and both telemetry parsers carry explicit per-metricdiff --git a/internal/api/resourceapi/truenas_core_reporting_test.go b/internal/api/resourceapi/truenas_core_reporting_test.gonew file mode 100644index 0000000000..7a659801fe--- /dev/null+++ b/internal/api/resourceapi/truenas_core_reporting_test.go@@ -0,0 +1,126 @@+package resourceapi++import (+ "context"+ "crypto/sha256"+ "encoding/json"+ "fmt"+ "net/http"+ "net/http/httptest"+ "strings"+ "testing"+ "time"++ "github.com/rcourtman/pulse-go-rewrite/internal/config"+ "github.com/rcourtman/pulse-go-rewrite/internal/models"+ "github.com/rcourtman/pulse-go-rewrite/internal/testutil/truenascore"+ "github.com/rcourtman/pulse-go-rewrite/internal/truenas"+ unified "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"+)++// This bridge models the CORE REST schema, not an appliance capture. The+// rejection of query is the reported #2964 response; reporting_query is the+// schema name, distinct from the second positional WebSocket argument. Keep+// this boundary independent of the client's request struct so changing both+// sides to the same wrong spelling cannot make the regression pass.+func TestCORE13RESTReportingQueryResourceJourney(t *testing.T) {+ previous := truenas.IsFeatureEnabled()+ truenas.SetFeatureEnabled(true)+ t.Cleanup(func() { truenas.SetFeatureEnabled(previous) })+ fixture := truenascore.NewRESTFixture(t)+ server := fixture.Server+ // Pin this fixture certificate through the production VerifyConnection+ // path; no public CA or trust bypass is needed or changed in the product.+ client, err := truenas.NewClient(truenas.ClientConfig{Host: server.URL, APIKey: "synthetic", InsecureSkipVerify: true, Fingerprint: fmt.Sprintf("%x", sha256.Sum256(server.Certificate().Raw))})+ if err != nil {+ t.Fatal(err)+ }+ t.Cleanup(client.Close)+ // Establish that the fixture really rejects the old envelope over HTTP.+ old, err := server.Client().Post(server.URL+"/api/v2.0/reporting/get_data", "application/json", strings.NewReader(`{"graphs":[{"name":"cpu"}],"query":{"start":1,"end":2}}`))+ if err != nil {+ t.Fatal(err)+ }+ _ = old.Body.Close()+ if old.StatusCode != 400 || fixture.RejectedOldKey.Load() != 1 {+ t.Fatal("fixture does not reproduce the reported query rejection")+ }+ provider := truenas.NewLiveProviderForConnection(&truenas.APIFetcher{Client: client}, "core-rest-fixture")+ for i, name := range []string{"complete", "rejected", "partial", "zero", "recovered"} {+ t.Run(name, func(t *testing.T) {+ fixture.Phase.Store(int64(i))+ beforePosts, beforeCatalogue := fixture.Posts.Load(), fixture.CatalogueCalls.Load()+ if err := provider.Refresh(context.Background()); err != nil {+ t.Fatal(err)+ }+ snapshot := provider.Snapshot()+ if len(snapshot.Disks) != 1 || len(snapshot.Pools) != 1 || snapshot.System.CPUCount != 8 || snapshot.System.MemoryTotalBytes != 16<<30 {+ t.Fatal("optional reporting failure discarded inventory or capacity")+ }+ registry := unified.NewRegistry(nil)+ registry.IngestRecords(unified.SourceTrueNAS, provider.Records())+ h := NewQueryService(&config.Config{DataPath: t.TempDir()})+ h.SetStateProvider(resourceUnifiedSeedProvider{snapshot: models.StateSnapshot{LastUpdate: time.Now()}, resources: registry.List()})+ rec := httptest.NewRecorder()+ h.HandleListResources(rec, httptest.NewRequest(http.MethodGet, "/api/resources?source=truenas&type=agent,storage,physical_disk&limit=100", nil))+ var response ResourcesResponse+ if rec.Code != 200 || json.Unmarshal(rec.Body.Bytes(), &response) != nil {+ t.Fatalf("resource HTTP response %d: %s", rec.Code, rec.Body.String())+ }+ var host *unified.Resource+ for index := range response.Data {+ if response.Data[index].Type == unified.ResourceTypeAgent {+ host = &response.Data[index]+ }+ }+ if host == nil || host.Metrics == nil || host.Metrics.Disk == nil || host.Metrics.Disk.Percent != 40 || host.Agent == nil || host.Agent.Memory == nil {+ t.Fatalf("host/storage capacity lost through HTTP: %s", rec.Body.String())+ }+ cpuKnown, memoryKnown := i != 1, i != 1 && i != 2+ if (host.Metrics.CPU != nil) != cpuKnown || (host.Metrics.Memory != nil) != memoryKnown || host.Agent.Memory.UsageUnavailable == memoryKnown {+ t.Fatalf("missing/partial readings became measured zero: %+v", host.Metrics)+ }+ if cpuKnown {+ want := 12.+ if i == 3 {+ want = 0+ }+ if host.Metrics.CPU.Percent != want {+ t.Fatalf("CPU = %v, want %v", host.Metrics.CPU.Percent, want)+ }+ }+ if memoryKnown {+ want := 37.5+ if i == 3 {+ want = 0+ }+ if host.Metrics.Memory.Percent != want {+ t.Fatalf("ARC-aware memory = %v, want %v", host.Metrics.Memory.Percent, want)+ }+ }+ id, history, historyErr := provider.SystemMetricHistory(context.Background(), time.Hour)+ wantCalls := int64(2) // one successful batch each for live and History+ if i == 1 {+ wantCalls = 14 // existing bounded optional-graph isolation+ if historyErr == nil || snapshot.System.Telemetry.ErrorCategory != "request_rejected" {+ t.Fatal("failed reporting disguised as successful History/telemetry")+ }+ } else if historyErr != nil || id != "core-rest-fixture" || len(history["cpu"]) != 2 {+ t.Fatalf("native CPU History lost: %s %+v %v", id, history, historyErr)+ }+ if fixture.Posts.Load()-beforePosts != wantCalls || fixture.CatalogueCalls.Load()-beforeCatalogue != 2 || fixture.RejectedOldKey.Load() != 1 {+ t.Fatal("unexpected request count, alias retry, or old query envelope")+ }+ for _, key := range []string{"memory", "diskread", "diskwrite", "netin", "netout"} {+ if (len(history[key]) > 0) != memoryKnown {+ t.Fatalf("History %s fabricated/lost a sibling: %+v", key, history[key])+ }+ }+ proof, err := json.Marshal(map[string]any{"phase": name, "resources": response, "history": history, "telemetry": snapshot.System.Telemetry})+ if err != nil {+ t.Fatal(err)+ }+ t.Logf("CORE_REST_BROWSER %s", proof)+ })+ }+}diff --git a/internal/api/truenas_core_history_integration_test.go b/internal/api/truenas_core_history_integration_test.gonew file mode 100644index 0000000000..444d83c74b--- /dev/null+++ b/internal/api/truenas_core_history_integration_test.go@@ -0,0 +1,282 @@+package api++import (+ "context"+ "crypto/sha256"+ "encoding/json"+ "fmt"+ "net/http"+ "net/http/httptest"+ "os"+ "path/filepath"+ "strings"+ "testing"+ "time"++ "github.com/rcourtman/pulse-go-rewrite/internal/config"+ "github.com/rcourtman/pulse-go-rewrite/internal/models"+ "github.com/rcourtman/pulse-go-rewrite/internal/monitoring"+ "github.com/rcourtman/pulse-go-rewrite/internal/testutil/truenascore"+ "github.com/rcourtman/pulse-go-rewrite/internal/truenas"+ "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"+)++// Complete the resource bridge at the actual authenticated chart route. Only+// the CORE server is synthetic: pinned TLS, REST decoding, provider, native+// poller fallback, monitor, tenant/scope checks and router are production code.+// No persistent database is needed for this already-supported native route.+func TestCORE13RESTReportingChartJourney(t *testing.T) {+ previous := truenas.IsFeatureEnabled()+ truenas.SetFeatureEnabled(true)+ t.Cleanup(func() { truenas.SetFeatureEnabled(previous) })+ previousMulti := IsMultiTenantEnabled()+ SetMultiTenantEnabled(true)+ t.Cleanup(func() { SetMultiTenantEnabled(previousMulti) })+ t.Setenv("PULSE_DEV", "true")+ fixture := truenascore.NewRESTFixture(t)+ fixture.WideHistory = true+ client, err := truenas.NewClient(truenas.ClientConfig{+ Host: fixture.Server.URL, APIKey: "synthetic-core-key", InsecureSkipVerify: true,+ Fingerprint: fmt.Sprintf("%x", sha256.Sum256(fixture.Server.Certificate().Raw)),+ })+ if err != nil {+ t.Fatal(err)+ }+ t.Cleanup(client.Close)+ provider := truenas.NewLiveProviderForConnection(&truenas.APIFetcher{Client: client}, "core-rest-fixture")+ poller := monitoring.NewTrueNASPoller(nil, time.Minute, nil)+ setUnexportedField(t, poller, "providersByOrg", map[string]map[string]*truenas.Provider{"org-core": {"core-rest-fixture": provider}})+ monitor := &monitoring.Monitor{}+ monitor.SetOrgID("org-core")+ setUnexportedField(t, monitor, "state", models.NewState())+ setUnexportedField(t, monitor, "metricsHistory", monitoring.NewMetricsHistory(20, time.Hour))+ const token = "core-history-read-fixture.12345678"+ readToken := newTokenRecord(t, token, []string{config.ScopeMonitoringRead}, nil)+ readToken.OrgID = "org-core"+ const wrongScope = "core-history-settings-fixture.12345678"+ scopeToken := newTokenRecord(t, wrongScope, []string{config.ScopeSettingsRead}, nil)+ scopeToken.OrgID = "org-core"+ cfg := newTestConfigWithTokens(t, readToken, scopeToken)+ cfg.AuthUser, cfg.AuthPass = "fixture-admin", "synthetic-password-hash"+ for _, org := range []string{"org-core", "org-other"} {+ if err := os.MkdirAll(filepath.Join(cfg.DataPath, "orgs", org), 0700); err != nil {+ t.Fatal(err)+ }+ }+ router := NewRouter(cfg, monitor, nil, nil, nil, "test")+ t.Cleanup(router.ShutdownBackgroundWorkers)+ t.Cleanup(router.ShutdownResourceStores)+ t.Cleanup(router.ShutdownRBAC)+ mtm := monitoring.NewMultiTenantMonitor(cfg, nil, nil)+ setUnexportedField(t, mtm, "monitors", map[string]*monitoring.Monitor{"org-core": monitor, "org-other": {}})+ router.mtMonitor = mtm+ // Match the production tenant resource binding as well as the chart+ // monitor binding; assigning the chart owner alone leaves resource reads+ // without their tenant snapshot provider.+ router.resourceHandlers.SetTenantStateProvider(NewMultiTenantStateProvider(mtm, monitor))+ router.setMonitorSupplementalRecordsProvider(unifiedresources.SourceTrueNAS, poller)+ const resourcePath = "/api/resources?source=truenas&type=agent,storage,physical_disk&limit=100"+ const historyPath = "/api/metrics-store/history?resourceType=agent&resourceId=core-rest-fixture&range=1h&maxPoints=240"+ request := func(raw, org, path string) *httptest.ResponseRecorder {+ req := httptest.NewRequest(http.MethodGet, path, nil)+ req.Header.Set("X-Pulse-Org-ID", org)+ if raw != "" {+ req.Header.Set("X-API-Token", raw)+ }+ rec := httptest.NewRecorder()+ router.Handler().ServeHTTP(rec, req)+ return rec+ }+ for _, test := range []struct {+ raw, org string+ want int+ }{+ {"", "org-core", 401}, {wrongScope, "org-core", 403}, {token, "org-other", 403},+ } {+ rec := request(test.raw, test.org, historyPath)+ if rec.Code != test.want || fixture.Posts.Load() != 0 {+ t.Fatalf("auth/scope/tenant boundary: status %d want %d native reads %d", rec.Code, test.want, fixture.Posts.Load())+ }+ }+ type point struct {+ Timestamp int64 `json:"timestamp"`+ Value, Min, Max float64+ }+ publish := func() {+ setUnexportedField(t, poller, "cachedRecordsByOrg", map[string]map[string][]unifiedresources.IngestRecord{"org-core": {"core-rest-fixture": provider.Records()}})+ monitor.SetResourceStoreAndProviders(unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil)), map[unifiedresources.DataSource]monitoring.MonitorSupplementalRecordsProvider{unifiedresources.SourceTrueNAS: poller})+ }+ var lastCollected time.Time+ for i, phase := range []string{"complete", "rejected", "partial", "zero", "recovered"} {+ t.Run(phase, func(t *testing.T) {+ // Independent cold-cache phases distinguish missing observations from+ // genuinely retained History. The warm-failure control below keeps+ // the existing cache intact instead of mistaking it for fresh data.+ setUnexportedField(t, monitor, "metricsHistory", monitoring.NewMetricsHistory(20, time.Hour))+ fixture.Phase.Store(int64(i))+ beforePosts := fixture.Posts.Load()+ if err := provider.Refresh(context.Background()); err != nil {+ t.Fatal(err)+ }+ snapshot := provider.Snapshot()+ if !snapshot.CollectedAt.After(lastCollected) {+ t.Fatal("collection stopped advancing")+ }+ lastCollected = snapshot.CollectedAt+ publish()+ resources := request(token, "org-core", resourcePath)+ var listed struct {+ Data []unifiedresources.Resource `json:"data"`+ }+ if resources.Code != 200 || json.Unmarshal(resources.Body.Bytes(), &listed) != nil {+ t.Fatalf("resources %d: %s", resources.Code, resources.Body.String())+ }+ var host *unifiedresources.Resource+ for j := range listed.Data {+ if listed.Data[j].Type == unifiedresources.ResourceTypeAgent {+ host = &listed.Data[j]+ }+ }+ if host == nil || host.MetricsTarget == nil || host.MetricsTarget.ResourceID != "core-rest-fixture" || host.Agent == nil || host.Agent.Memory == nil {+ t.Fatalf("canonical host/target lost: %s", resources.Body.String())+ }+ memory := host.Agent.Memory+ knownCPU, knownMemory := i != 1, i != 1 && i != 2+ if (host.Metrics.CPU != nil) != knownCPU || (host.Metrics.Memory != nil) != knownMemory || memory.UsageUnavailable == knownMemory {+ t.Fatalf("unknown became idle: %+v %+v", host.Metrics, memory)+ }+ if memory.Total != 16<<30 || host.Metrics.Disk == nil || host.Metrics.Disk.Percent != 40 {+ t.Fatal("capacity was lost or substituted for telemetry")+ }+ if knownMemory {+ used, free, cache := int64(6<<30), int64(2<<30), int64(8<<30)+ if i == 3 {+ used, free, cache = 0, 16<<30, 0+ }+ if memory.Used != used || memory.Free != free || memory.Cache != cache {+ t.Fatalf("free/ARC/effective used changed: %+v", memory)+ }+ }+ history24 := request(token, "org-core", strings.Replace(historyPath, "range=1h", "range=24h", 1))+ history := request(token, "org-core", historyPath)+ wantCalls := int64(3)+ if i == 1 {+ wantCalls = 21+ }+ if fixture.Posts.Load()-beforePosts != wantCalls || fixture.RejectedOldKey.Load() != 0 {+ t.Fatal("chart did not traverse native reporting, or repeated the rejected envelope")+ }+ if history24.Code != history.Code {+ t.Fatal("default-range chart status disagrees with selected range")+ }+ if i == 1 && snapshot.System.Telemetry.ErrorCategory != "request_rejected" {+ t.Fatal("reporting rejection lost its diagnostic category")+ }+ var data struct {+ ResourceType, ResourceID string+ Metrics map[string][]point+ }+ if history.Code != 200 || json.Unmarshal(history.Body.Bytes(), &data) != nil || data.ResourceType != "agent" || data.ResourceID != "core-rest-fixture" {+ t.Fatalf("chart response %d: %s", history.Code, history.Body.String())+ }+ expected := map[string]float64{"cpu": 12, "memory": 37.5, "diskread": 1024, "diskwrite": 2048, "netin": 32, "netout": 64, "temperature": 42}+ for metric, value := range expected {+ known := knownMemory || (metric == "cpu" && knownCPU)+ if !known {+ if len(data.Metrics[metric]) != 0 {+ t.Fatalf("cold chart fabricated %s: %+v", metric, data.Metrics[metric])+ }+ continue+ }+ if i == 3 && metric != "temperature" {+ value = 0+ }+ points := data.Metrics[metric]+ if len(points) != 4 {+ t.Fatalf("native %s observations lost: %+v", metric, points)+ }+ for j, p := range points {+ if p.Value != value || p.Min != value || p.Max != value || (j > 0 && p.Timestamp <= points[j-1].Timestamp) {+ t.Fatalf("native sample/bounds/order changed for %s: %+v", metric, points)+ }+ }+ if points[3].Timestamp-points[0].Timestamp < 57*60*1000 {+ t.Fatal("hour chart became a current-value shortcut")+ }+ }+ // Pool capacity is independent of reporting and remains valid even+ // when every optional CPU/memory/IO graph is rejected.+ if len(data.Metrics["disk"]) != 1 || data.Metrics["disk"][0].Value != 40 {+ t.Fatal("reporting failure erased independent capacity history")+ }++ payload, err := json.Marshal(map[string]any{"phase": phase, "resources": json.RawMessage(resources.Body.Bytes()), "history": map[string]any{"status": history.Code, "body": json.RawMessage(history.Body.Bytes())}, "history24": map[string]any{"status": history24.Code, "body": json.RawMessage(history24.Body.Bytes())}, "telemetry": snapshot.System.Telemetry})+ if err != nil {+ t.Fatal(err)+ }+ t.Logf("CORE_CHART_BROWSER %s", payload)+ if i == 0 {+ retained := monitor.GetGuestMetrics("agent:core-rest-fixture", time.Hour)+ before := fixture.Posts.Load()+ fixture.Phase.Store(1)+ if err := provider.Refresh(context.Background()); err != nil {+ t.Fatal(err)+ }+ publish()+ resources := request(token, "org-core", resourcePath)+ var current struct{ Data []unifiedresources.Resource }+ if resources.Code != 200 || json.Unmarshal(resources.Body.Bytes(), ¤t) != nil {+ t.Fatal("warm resource read failed")+ }+ for _, row := range current.Data {+ if row.Type == unifiedresources.ResourceTypeAgent && (row.Metrics.CPU != nil || row.Metrics.Memory != nil) {+ t.Fatal("retained History became current telemetry")+ }+ }+ history24 := request(token, "org-core", strings.Replace(historyPath, "range=1h", "range=24h", 1))+ history := request(token, "org-core", historyPath)+ var warm struct{ Metrics map[string][]point }+ if history.Code != 200 || history24.Code != 200 || json.Unmarshal(history.Body.Bytes(), &warm) != nil || fixture.Posts.Load()-before != 21 {+ t.Fatal("warm native failure discarded valid History or escaped its request bound")+ }+ if len(retained["cpu"]) != 1 || len(retained["memory"]) != 1 {+ t.Fatal("warm control has no original CPU/memory observations")+ }+ for metric, points := range retained {+ if metric == "disk" {+ continue+ } // capacity was collected again; it is not a retained optional reading+ if len(points) == 0 {+ if len(warm.Metrics[metric]) != 0 {+ t.Fatalf("warm failure invented unobserved %s", metric)+ }+ continue+ }++ if len(points) != 1 || len(warm.Metrics[metric]) != 1 {+ t.Fatalf("warm %s history changed cardinality", metric)+ }+ old, got := points[0], warm.Metrics[metric][0]+ wantTime := (old.Timestamp.Unix()/15*15 + 7) * 1000+ if got.Timestamp != wantTime || got.Value != old.Value || got.Min != old.Value || got.Max != old.Value {+ t.Fatalf("warm %s changed observation time/value: old %+v, returned %+v", metric, old, got)+ }+ }+ payload, err := json.Marshal(map[string]any{"phase": "retained", "resources": json.RawMessage(resources.Body.Bytes()), "history": map[string]any{"status": history.Code, "body": json.RawMessage(history.Body.Bytes())}, "history24": map[string]any{"status": history24.Code, "body": json.RawMessage(history24.Body.Bytes())}, "telemetry": provider.Snapshot().System.Telemetry})+ if err != nil {+ t.Fatal(err)+ }+ t.Logf("CORE_CHART_BROWSER %s", payload)+ }++ })+ }+ before := fixture.Posts.Load()+ if cfg.RemoveAPIToken(readToken.ID) == nil {+ t.Fatal("fixture token missing")+ }+ cfg.SortAPITokens()+ if rec := request(token, "org-core", historyPath); rec.Code != 401 || fixture.Posts.Load() != before {+ t.Fatal("revoked token still reached native History")+ }+}diff --git a/internal/monitoring/canonical_guardrails_test.go b/internal/monitoring/canonical_guardrails_test.goindex 45c59322b4..7531c0d5e3 100644--- a/internal/monitoring/canonical_guardrails_test.go+++ b/internal/monitoring/canonical_guardrails_test.go@@ -3473,3 +3473,34 @@ func TestVMwareInventoryRefreshBudgetContract(t *testing.T) {t.Run("configuration", testVMwarePollTimeoutConfiguration)t.Run("refresh-and-projection", testVMwarePollRefreshBudgetAndProjection)}++// The REST seed must preserve the same resolved coordinates as the broadcast+// generation, without consulting a later registry or editing a shared view.+func TestUnifiedSnapshotPreservesCapturedHistoryTargets(t *testing.T) {+ oldTarget := &unifiedresources.MetricsTarget{ResourceType: "agent", ResourceID: "old-connection"}+ rows := []unifiedresources.Resource{+ {ID: "canonical-row", Type: unifiedresources.ResourceTypeAgent, MetricsTarget: oldTarget},+ {ID: "unknown-row", Type: unifiedresources.ResourceTypeAgent, MetricsTarget: oldTarget},+ }+ targets := projectionMetricsTargets{"canonical-row": {ResourceType: "agent", ResourceID: "native-connection"}}+ view := monitorUnifiedStateView{resources: rows, metricsTargets: targets}+ got := unifiedSnapshotResources(view)+ if got[0].MetricsTarget == nil || got[0].MetricsTarget.ResourceID != "native-connection" || got[1].MetricsTarget != nil {+ t.Fatalf("captured known/unknown targets lost: %+v", got)+ }+ if rows[0].MetricsTarget != oldTarget || rows[1].MetricsTarget != oldTarget || oldTarget.ResourceID != "old-connection" {+ t.Fatal("snapshot decorated the shared read-state rows")+ }+ got[0].MetricsTarget.ResourceID = "caller-change"+ got[0].Name = "caller-change"+ again := unifiedSnapshotResources(view)+ if again[0].Name != "" || again[0].MetricsTarget.ResourceID != "native-connection" || targets["canonical-row"].ResourceID != "native-connection" {+ t.Fatal("returned snapshot changed the captured generation")+ }+ // A store without a captured target projection retains its existing+ // source row; the snapshot does not invent a coordinate from its name.+ fallback := unifiedSnapshotResources(monitorUnifiedStateView{resources: rows})+ if fallback[0].MetricsTarget != oldTarget {+ t.Fatal("projection-less fallback lost its supplied coordinate")+ }+}diff --git a/internal/monitoring/resource_stale_thresholds.go b/internal/monitoring/resource_stale_thresholds.goindex 7f51b37fe9..9a0d85f25d 100644--- a/internal/monitoring/resource_stale_thresholds.go+++ b/internal/monitoring/resource_stale_thresholds.go@@ -140,7 +140,26 @@ func resourceStaleThresholdsForIntervals(func (m *Monitor) UnifiedResourceSnapshotWithStaleThresholds() ([]unifiedresources.Resource, time.Time, map[unifiedresources.DataSource]time.Duration) {m.refreshUnifiedResourceRead()view := m.currentUnifiedStateView()- return m.applyPersistedMetadataToUnifiedResources(view.resources), view.freshness, view.staleThresholds+ return m.applyPersistedMetadataToUnifiedResources(unifiedSnapshotResources(view)), view.freshness, view.staleThresholds+}++// unifiedSnapshotResources carries the captured generation's resolved History+// coordinates across the REST registry rebuild. Source-native IDs cannot always+// be reconstructed from presentation rows (notably TrueNAS and VMware). The+// targets already belong to this read, so never consult the live store again or+// guess from a display name. Copy top-level rows before decorating: mock views+// and cached read-state overlays may share their resource slice with readers.+func unifiedSnapshotResources(view monitorUnifiedStateView) []unifiedresources.Resource {+ if view.metricsTargets == nil {+ return view.resources+ }+ resources := append([]unifiedresources.Resource(nil), view.resources...)+ for i := range resources {+ // Missing in this captured generation is authoritative too; do not+ // carry a stale target from an earlier presentation row.+ resources[i].MetricsTarget = view.metricsTargets.MetricsTargetForResource(resources[i].ID)+ }+ return resources}// Headless readers need the same publication freshness as frontend state reads.diff --git a/internal/monitoring/truenas_poller_test.go b/internal/monitoring/truenas_poller_test.goindex eedb2c6eb4..7af12fa8f8 100644--- a/internal/monitoring/truenas_poller_test.go+++ b/internal/monitoring/truenas_poller_test.go@@ -2544,7 +2544,7 @@ func TestTrueNASPartialReportingPipeline(t *testing.T) {break}var query struct {- Query map[string]any `json:"query"`+ Query map[string]any `json:"reporting_query"`Graphs []map[string]any `json:"graphs"`}if err := json.NewDecoder(r.Body).Decode(&query); err != nil {diff --git a/internal/testutil/truenascore/rest.go b/internal/testutil/truenascore/rest.gonew file mode 100644index 0000000000..e77f27373e--- /dev/null+++ b/internal/testutil/truenascore/rest.go@@ -0,0 +1,131 @@+// Package truenascore supplies the explicit CORE 13 REST schema used by+// read-only client/resource/chart integration tests. It is not an appliance.+package truenascore++import (+ "encoding/json"+ "fmt"+ "net/http"+ "net/http/httptest"+ "sync/atomic"+ "testing"+)++// RESTFixture rejects the reported old request key independently of the+// production request type. Phase: complete, rejected, CPU-only, zero, recovered.+// Set WideHistory before the first request to return four native observations+// spanning the requested hour, rather than two live observations ten seconds apart.+type RESTFixture struct {+ Server *httptest.Server+ Phase, Posts, CatalogueCalls, RejectedOldKey atomic.Int64+ WideHistory bool+}++func NewRESTFixture(t *testing.T) *RESTFixture {+ t.Helper()+ f := &RESTFixture{}+ f.Server = httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ switch r.URL.Path {+ case "/api/v2.0/system/info":+ fmt.Fprint(w, `{"hostname":"core-fixture","version":"TrueNAS-13.0-U6.8","cores":8,"physmem":17179869184}`)+ case "/api/v2.0/pool":+ fmt.Fprint(w, `[{"id":1,"name":"tank","status":"ONLINE","size":1000,"allocated":400,"free":600}]`)+ case "/api/v2.0/disk":+ fmt.Fprint(w, `[{"identifier":"disk-a","name":"da0","serial":"fixture-disk","rotationrate":"unknown","size":1000}]`)+ case "/api/v2.0/pool/dataset", "/api/v2.0/alert/list":+ fmt.Fprint(w, `[]`)+ case "/api/v2.0/reporting/graphs":+ f.CatalogueCalls.Add(1)+ fmt.Fprint(w, `[{"name":"disk","identifiers":["da0"]},{"name":"interface","identifiers":["vtnet0"]}]`)+ case "/api/v2.0/reporting/get_data":+ f.Posts.Add(1)+ var body map[string]json.RawMessage+ if r.Method != http.MethodPost || json.NewDecoder(r.Body).Decode(&body) != nil {+ http.Error(w, "invalid fixture request", http.StatusBadRequest)+ return+ }+ if _, present := body["query"]; present {+ f.RejectedOldKey.Add(1)+ w.WriteHeader(http.StatusBadRequest)+ fmt.Fprint(w, `{"message":"The following attributes are not expected: query"}`)+ return+ }+ var graphs []struct{ Name, Identifier string }+ var window struct {+ Start, End int64+ Aggregate bool+ }+ if len(body) != 2 || json.Unmarshal(body["graphs"], &graphs) != nil || json.Unmarshal(body["reporting_query"], &window) != nil || !window.Aggregate || window.Start >= window.End {+ http.Error(w, "missing graphs/reporting_query range", http.StatusBadRequest)+ return+ }+ if f.Phase.Load() == 1 {+ w.WriteHeader(http.StatusBadRequest)+ fmt.Fprint(w, `{"message":"synthetic reporting validation failure"}`)+ return+ }+ var result []map[string]any+ for _, graph := range graphs {+ var legend []string+ var values []any+ zero := f.Phase.Load() == 3+ if f.Phase.Load() == 2 && graph.Name != "cpu" {+ continue // usable CPU does not establish any sibling reading+ }+ switch graph.Name {+ case "cpu":+ legend, values = []string{"user", "nice", "system", "interrupt", "idle"}, []any{8, 0, 3, 1, 88}+ if zero {+ values = []any{0, 0, 0, 0, 100}+ }+ case "memory":+ legend = []string{"memory-active_value", "memory-inactive_value", "memory-wired_value", "memory-laundry_value", "memory-free_value"}+ values = []any{1 << 30, 2 << 30, 11 << 30, 0, 2 << 30}+ if zero {+ values = []any{0, 0, 0, 0, 16 << 30}+ }+ case "arcsize":+ legend, values = []string{"arcsize_value"}, []any{8 << 30}+ if zero {+ values = []any{0}+ }+ case "disk":+ if graph.Identifier != "da0" {+ t.Error("disk identifier lost")+ }+ legend, values = []string{"disk_octets_read", "disk_octets_write"}, []any{1024, 2048}+ if zero {+ values = []any{0, 0}+ }+ case "interface":+ if graph.Identifier != "vtnet0" {+ t.Error("interface identifier lost")+ }+ legend, values = []string{"rx", "tx", "overlap"}, []any{32, 64, 32}+ if zero {+ values = []any{0, 0, 0}+ }+ case "cputemp":+ legend, values = []string{"temperature"}, []any{42}+ default:+ t.Errorf("unexpected graph %s", graph.Name)+ }+ start, step := window.End-10, int64(10)+ data := []any{values, values}+ if f.WideHistory && window.End-window.Start >= 3600 {+ start, step = window.End-3540, 1160+ data = []any{values, values, values, values}+ }+ result = append(result, map[string]any{"name": graph.Name, "identifier": graph.Identifier, "legend": legend, "start": start, "end": window.End, "step": step, "data": data})+ }+ if err := json.NewEncoder(w).Encode(result); err != nil {+ t.Error(err)+ }+ default:+ http.NotFound(w, r)+ }+ }))+ t.Cleanup(f.Server.Close)+ return f+}diff --git a/internal/truenas/client.go b/internal/truenas/client.goindex ec12697deb..47b98c3409 100644--- a/internal/truenas/client.go+++ b/internal/truenas/client.go@@ -480,13 +480,14 @@ func isReportingGraphFailure(err error) bool {}// getReportingDataREST issues reporting.get_data over the REST v2.0 transport.-// The middleware method takes positional parameters (graphs, query); REST-// serves it as a POST body keyed by parameter name.+// WebSocket parameters are positional, but REST v2.0 binds top-level keys to+// the middleware schema names: graphs and reporting_query, not the Go/RPC+// variable name query. CORE rejects an unknown query key before reading graphs.func (c *Client) getReportingDataREST(ctx context.Context, graphs []map[string]any, query map[string]any) ([]trueNASReportingGetDataResponse, error) {var response []trueNASReportingGetDataResponseerr := c.postJSON(ctx, "/reporting/get_data", map[string]any{- "graphs": graphs,- "query": query,+ "graphs": graphs,+ "reporting_query": query,}, &response)if err != nil {return nil, errdiff --git a/internal/truenas/client_test.go b/internal/truenas/client_test.goindex e1f305cbc2..d2446ec2aa 100644--- a/internal/truenas/client_test.go+++ b/internal/truenas/client_test.go@@ -2134,7 +2134,7 @@ func (s *isolatedReportingTransport) RoundTrip(r *http.Request) (*http.Response,s.calls++var request struct {Graphs []map[string]any `json:"graphs"`- Query map[string]any `json:"query"`+ Query map[string]any `json:"reporting_query"`}if err := json.NewDecoder(r.Body).Decode(&request); err != nil {return nil, err@@ -2300,7 +2300,7 @@ func (s *nativeShapeReportingTransport) RoundTrip(r *http.Request) (*http.Respon}var request struct {Graphs []map[string]any `json:"graphs"`- Query map[string]any `json:"query"`+ Query map[string]any `json:"reporting_query"`}if err := json.NewDecoder(r.Body).Decode(&request); err != nil {return nil, errdiff --git a/internal/truenas/core_reporting_test.go b/internal/truenas/core_reporting_test.goindex 08a2594fde..6a6bddfffd 100644--- a/internal/truenas/core_reporting_test.go+++ b/internal/truenas/core_reporting_test.go@@ -243,10 +243,15 @@ func (s *core13ReportingTransport) RoundTrip(r *http.Request) (*http.Response, es.reportingCalls++var request struct {Graphs []map[string]any `json:"graphs"`- Query map[string]any `json:"query"`+ Query map[string]any `json:"reporting_query"`}- if err := json.NewDecoder(r.Body).Decode(&request); err != nil {- return nil, err+ decoder := json.NewDecoder(r.Body)+ decoder.DisallowUnknownFields()+ if err := decoder.Decode(&request); err != nil {+ return nil, fmt.Errorf("invalid CORE REST reporting envelope: %w", err)+ }+ if len(request.Graphs) == 0 || request.Query["aggregate"] != true {+ return nil, fmt.Errorf("missing graphs or reporting_query range")}s.requested = append(s.requested, request.Graphs...)var responses []trueNASReportingGetDataResponsediff --git a/internal/truenas/reporting_aggregation_test.go b/internal/truenas/reporting_aggregation_test.goindex 1fcecb8b7a..286ab6eec1 100644--- a/internal/truenas/reporting_aggregation_test.go+++ b/internal/truenas/reporting_aggregation_test.go@@ -227,7 +227,7 @@ func TestRESTReportingDefaultAggregationsPreserveSamples(t *testing.T) {calls++var request struct {Graphs []map[string]any `json:"graphs"`- Query map[string]any `json:"query"`+ Query map[string]any `json:"reporting_query"`}if err := json.NewDecoder(r.Body).Decode(&request); err != nil {return nil, err