Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Restore CORE reporting and native History on current main

pulse-triage[bot]4771e3ace1cb
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index a89ad201c2..ba0ab40318 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -2006,8 +2006,8 @@ are not native appliance or containing-release acceptance.
A connection proven to run a recognized legacy CORE/FreeNAS release has no
`reporting.realtime` JSON-RPC subscription, so live system telemetry and
host-chart history are reconstructed from the REST reporting API
-(`reporting.get_data`, served as a POST endpoint keyed by its `graphs` and
-`query` parameters). The reporting `memory` and `arcsize` graphs supply the
+(`reporting.get_data`, served as a POST endpoint keyed by the middleware
+schema names `graphs` and `reporting_query`). The reporting `memory` and `arcsize` graphs supply the
available and ARC readings that the JSON-RPC realtime subscription otherwise
provides. When no available-memory reading can be established, the provider
must not derive usage from an absent available value: the system memory metric is
@@ -2024,6 +2024,30 @@ confirmation.
+### CORE REST envelope and captured History targets — issues #2077 and #2964
+
+Legacy REST `POST /reporting/get_data` sends exactly `graphs` and
+`reporting_query`. WebSocket's positional second argument is not a REST key
+named `query`: CORE rejects that unknown attribute before collecting any graph.
+Live reads and native History share this envelope and unchanged aggregate,
+start/end range, graph catalogue and optional-graph isolation. TLS, authentication,
+transport selection and per-metric zero/unknown semantics are unchanged.
+
+The monitor's resource snapshot carries its captured generation's resolved
+History target into the detached REST seed, including authoritative absence.
+It never guesses a native connection ID from a display name, consults a later
+registry generation or decorates shared view rows. This restores the existing
+canonical History identity contract rather than adding a route or schema.
+
+`TestCORE13RESTReportingQueryResourceJourney` and
+`TestCORE13RESTReportingChartJourney` exercise the real pinned-TLS
+client/provider, authenticated resources and chart route, native History,
+free/ARC memory, unavailable/partial/zero/recovered readings, retained historical
+observations and access revocation. `TestUnifiedSnapshotPreservesCapturedHistoryTargets`
+protects captured coordinates, authoritative absence and detached ownership.
+These schema-faithful synthetic controls are not native appliance acceptance,
+reporter confirmation, data-pool recovery or release availability.
+
### TrueNAS partial samples and nonblocking telemetry diagnostics — issue #2077
Live system identity and both telemetry parsers carry explicit per-metric
diff --git a/internal/api/resourceapi/truenas_core_reporting_test.go b/internal/api/resourceapi/truenas_core_reporting_test.go
new file mode 100644
index 0000000000..7a659801fe
--- /dev/null
+++ b/internal/api/resourceapi/truenas_core_reporting_test.go
@@ -0,0 +1,126 @@
+package resourceapi
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/config"
+ "github.com/rcourtman/pulse-go-rewrite/internal/models"
+ "github.com/rcourtman/pulse-go-rewrite/internal/testutil/truenascore"
+ "github.com/rcourtman/pulse-go-rewrite/internal/truenas"
+ unified "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
+)
+
+// This bridge models the CORE REST schema, not an appliance capture. The
+// rejection of query is the reported #2964 response; reporting_query is the
+// schema name, distinct from the second positional WebSocket argument. Keep
+// this boundary independent of the client's request struct so changing both
+// sides to the same wrong spelling cannot make the regression pass.
+func TestCORE13RESTReportingQueryResourceJourney(t *testing.T) {
+ previous := truenas.IsFeatureEnabled()
+ truenas.SetFeatureEnabled(true)
+ t.Cleanup(func() { truenas.SetFeatureEnabled(previous) })
+ fixture := truenascore.NewRESTFixture(t)
+ server := fixture.Server
+ // Pin this fixture certificate through the production VerifyConnection
+ // path; no public CA or trust bypass is needed or changed in the product.
+ client, err := truenas.NewClient(truenas.ClientConfig{Host: server.URL, APIKey: "synthetic", InsecureSkipVerify: true, Fingerprint: fmt.Sprintf("%x", sha256.Sum256(server.Certificate().Raw))})
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(client.Close)
+ // Establish that the fixture really rejects the old envelope over HTTP.
+ old, err := server.Client().Post(server.URL+"/api/v2.0/reporting/get_data", "application/json", strings.NewReader(`{"graphs":[{"name":"cpu"}],"query":{"start":1,"end":2}}`))
+ if err != nil {
+ t.Fatal(err)
+ }
+ _ = old.Body.Close()
+ if old.StatusCode != 400 || fixture.RejectedOldKey.Load() != 1 {
+ t.Fatal("fixture does not reproduce the reported query rejection")
+ }
+ provider := truenas.NewLiveProviderForConnection(&truenas.APIFetcher{Client: client}, "core-rest-fixture")
+ for i, name := range []string{"complete", "rejected", "partial", "zero", "recovered"} {
+ t.Run(name, func(t *testing.T) {
+ fixture.Phase.Store(int64(i))
+ beforePosts, beforeCatalogue := fixture.Posts.Load(), fixture.CatalogueCalls.Load()
+ if err := provider.Refresh(context.Background()); err != nil {
+ t.Fatal(err)
+ }
+ snapshot := provider.Snapshot()
+ if len(snapshot.Disks) != 1 || len(snapshot.Pools) != 1 || snapshot.System.CPUCount != 8 || snapshot.System.MemoryTotalBytes != 16<<30 {
+ t.Fatal("optional reporting failure discarded inventory or capacity")
+ }
+ registry := unified.NewRegistry(nil)
+ registry.IngestRecords(unified.SourceTrueNAS, provider.Records())
+ h := NewQueryService(&config.Config{DataPath: t.TempDir()})
+ h.SetStateProvider(resourceUnifiedSeedProvider{snapshot: models.StateSnapshot{LastUpdate: time.Now()}, resources: registry.List()})
+ rec := httptest.NewRecorder()
+ h.HandleListResources(rec, httptest.NewRequest(http.MethodGet, "/api/resources?source=truenas&type=agent,storage,physical_disk&limit=100", nil))
+ var response ResourcesResponse
+ if rec.Code != 200 || json.Unmarshal(rec.Body.Bytes(), &response) != nil {
+ t.Fatalf("resource HTTP response %d: %s", rec.Code, rec.Body.String())
+ }
+ var host *unified.Resource
+ for index := range response.Data {
+ if response.Data[index].Type == unified.ResourceTypeAgent {
+ host = &response.Data[index]
+ }
+ }
+ if host == nil || host.Metrics == nil || host.Metrics.Disk == nil || host.Metrics.Disk.Percent != 40 || host.Agent == nil || host.Agent.Memory == nil {
+ t.Fatalf("host/storage capacity lost through HTTP: %s", rec.Body.String())
+ }
+ cpuKnown, memoryKnown := i != 1, i != 1 && i != 2
+ if (host.Metrics.CPU != nil) != cpuKnown || (host.Metrics.Memory != nil) != memoryKnown || host.Agent.Memory.UsageUnavailable == memoryKnown {
+ t.Fatalf("missing/partial readings became measured zero: %+v", host.Metrics)
+ }
+ if cpuKnown {
+ want := 12.
+ if i == 3 {
+ want = 0
+ }
+ if host.Metrics.CPU.Percent != want {
+ t.Fatalf("CPU = %v, want %v", host.Metrics.CPU.Percent, want)
+ }
+ }
+ if memoryKnown {
+ want := 37.5
+ if i == 3 {
+ want = 0
+ }
+ if host.Metrics.Memory.Percent != want {
+ t.Fatalf("ARC-aware memory = %v, want %v", host.Metrics.Memory.Percent, want)
+ }
+ }
+ id, history, historyErr := provider.SystemMetricHistory(context.Background(), time.Hour)
+ wantCalls := int64(2) // one successful batch each for live and History
+ if i == 1 {
+ wantCalls = 14 // existing bounded optional-graph isolation
+ if historyErr == nil || snapshot.System.Telemetry.ErrorCategory != "request_rejected" {
+ t.Fatal("failed reporting disguised as successful History/telemetry")
+ }
+ } else if historyErr != nil || id != "core-rest-fixture" || len(history["cpu"]) != 2 {
+ t.Fatalf("native CPU History lost: %s %+v %v", id, history, historyErr)
+ }
+ if fixture.Posts.Load()-beforePosts != wantCalls || fixture.CatalogueCalls.Load()-beforeCatalogue != 2 || fixture.RejectedOldKey.Load() != 1 {
+ t.Fatal("unexpected request count, alias retry, or old query envelope")
+ }
+ for _, key := range []string{"memory", "diskread", "diskwrite", "netin", "netout"} {
+ if (len(history[key]) > 0) != memoryKnown {
+ t.Fatalf("History %s fabricated/lost a sibling: %+v", key, history[key])
+ }
+ }
+ proof, err := json.Marshal(map[string]any{"phase": name, "resources": response, "history": history, "telemetry": snapshot.System.Telemetry})
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Logf("CORE_REST_BROWSER %s", proof)
+ })
+ }
+}
diff --git a/internal/api/truenas_core_history_integration_test.go b/internal/api/truenas_core_history_integration_test.go
new file mode 100644
index 0000000000..444d83c74b
--- /dev/null
+++ b/internal/api/truenas_core_history_integration_test.go
@@ -0,0 +1,282 @@
+package api
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/config"
+ "github.com/rcourtman/pulse-go-rewrite/internal/models"
+ "github.com/rcourtman/pulse-go-rewrite/internal/monitoring"
+ "github.com/rcourtman/pulse-go-rewrite/internal/testutil/truenascore"
+ "github.com/rcourtman/pulse-go-rewrite/internal/truenas"
+ "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
+)
+
+// Complete the resource bridge at the actual authenticated chart route. Only
+// the CORE server is synthetic: pinned TLS, REST decoding, provider, native
+// poller fallback, monitor, tenant/scope checks and router are production code.
+// No persistent database is needed for this already-supported native route.
+func TestCORE13RESTReportingChartJourney(t *testing.T) {
+ previous := truenas.IsFeatureEnabled()
+ truenas.SetFeatureEnabled(true)
+ t.Cleanup(func() { truenas.SetFeatureEnabled(previous) })
+ previousMulti := IsMultiTenantEnabled()
+ SetMultiTenantEnabled(true)
+ t.Cleanup(func() { SetMultiTenantEnabled(previousMulti) })
+ t.Setenv("PULSE_DEV", "true")
+ fixture := truenascore.NewRESTFixture(t)
+ fixture.WideHistory = true
+ client, err := truenas.NewClient(truenas.ClientConfig{
+ Host: fixture.Server.URL, APIKey: "synthetic-core-key", InsecureSkipVerify: true,
+ Fingerprint: fmt.Sprintf("%x", sha256.Sum256(fixture.Server.Certificate().Raw)),
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(client.Close)
+ provider := truenas.NewLiveProviderForConnection(&truenas.APIFetcher{Client: client}, "core-rest-fixture")
+ poller := monitoring.NewTrueNASPoller(nil, time.Minute, nil)
+ setUnexportedField(t, poller, "providersByOrg", map[string]map[string]*truenas.Provider{"org-core": {"core-rest-fixture": provider}})
+ monitor := &monitoring.Monitor{}
+ monitor.SetOrgID("org-core")
+ setUnexportedField(t, monitor, "state", models.NewState())
+ setUnexportedField(t, monitor, "metricsHistory", monitoring.NewMetricsHistory(20, time.Hour))
+ const token = "core-history-read-fixture.12345678"
+ readToken := newTokenRecord(t, token, []string{config.ScopeMonitoringRead}, nil)
+ readToken.OrgID = "org-core"
+ const wrongScope = "core-history-settings-fixture.12345678"
+ scopeToken := newTokenRecord(t, wrongScope, []string{config.ScopeSettingsRead}, nil)
+ scopeToken.OrgID = "org-core"
+ cfg := newTestConfigWithTokens(t, readToken, scopeToken)
+ cfg.AuthUser, cfg.AuthPass = "fixture-admin", "synthetic-password-hash"
+ for _, org := range []string{"org-core", "org-other"} {
+ if err := os.MkdirAll(filepath.Join(cfg.DataPath, "orgs", org), 0700); err != nil {
+ t.Fatal(err)
+ }
+ }
+ router := NewRouter(cfg, monitor, nil, nil, nil, "test")
+ t.Cleanup(router.ShutdownBackgroundWorkers)
+ t.Cleanup(router.ShutdownResourceStores)
+ t.Cleanup(router.ShutdownRBAC)
+ mtm := monitoring.NewMultiTenantMonitor(cfg, nil, nil)
+ setUnexportedField(t, mtm, "monitors", map[string]*monitoring.Monitor{"org-core": monitor, "org-other": {}})
+ router.mtMonitor = mtm
+ // Match the production tenant resource binding as well as the chart
+ // monitor binding; assigning the chart owner alone leaves resource reads
+ // without their tenant snapshot provider.
+ router.resourceHandlers.SetTenantStateProvider(NewMultiTenantStateProvider(mtm, monitor))
+ router.setMonitorSupplementalRecordsProvider(unifiedresources.SourceTrueNAS, poller)
+ const resourcePath = "/api/resources?source=truenas&type=agent,storage,physical_disk&limit=100"
+ const historyPath = "/api/metrics-store/history?resourceType=agent&resourceId=core-rest-fixture&range=1h&maxPoints=240"
+ request := func(raw, org, path string) *httptest.ResponseRecorder {
+ req := httptest.NewRequest(http.MethodGet, path, nil)
+ req.Header.Set("X-Pulse-Org-ID", org)
+ if raw != "" {
+ req.Header.Set("X-API-Token", raw)
+ }
+ rec := httptest.NewRecorder()
+ router.Handler().ServeHTTP(rec, req)
+ return rec
+ }
+ for _, test := range []struct {
+ raw, org string
+ want int
+ }{
+ {"", "org-core", 401}, {wrongScope, "org-core", 403}, {token, "org-other", 403},
+ } {
+ rec := request(test.raw, test.org, historyPath)
+ if rec.Code != test.want || fixture.Posts.Load() != 0 {
+ t.Fatalf("auth/scope/tenant boundary: status %d want %d native reads %d", rec.Code, test.want, fixture.Posts.Load())
+ }
+ }
+ type point struct {
+ Timestamp int64 `json:"timestamp"`
+ Value, Min, Max float64
+ }
+ publish := func() {
+ setUnexportedField(t, poller, "cachedRecordsByOrg", map[string]map[string][]unifiedresources.IngestRecord{"org-core": {"core-rest-fixture": provider.Records()}})
+ monitor.SetResourceStoreAndProviders(unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil)), map[unifiedresources.DataSource]monitoring.MonitorSupplementalRecordsProvider{unifiedresources.SourceTrueNAS: poller})
+ }
+ var lastCollected time.Time
+ for i, phase := range []string{"complete", "rejected", "partial", "zero", "recovered"} {
+ t.Run(phase, func(t *testing.T) {
+ // Independent cold-cache phases distinguish missing observations from
+ // genuinely retained History. The warm-failure control below keeps
+ // the existing cache intact instead of mistaking it for fresh data.
+ setUnexportedField(t, monitor, "metricsHistory", monitoring.NewMetricsHistory(20, time.Hour))
+ fixture.Phase.Store(int64(i))
+ beforePosts := fixture.Posts.Load()
+ if err := provider.Refresh(context.Background()); err != nil {
+ t.Fatal(err)
+ }
+ snapshot := provider.Snapshot()
+ if !snapshot.CollectedAt.After(lastCollected) {
+ t.Fatal("collection stopped advancing")
+ }
+ lastCollected = snapshot.CollectedAt
+ publish()
+ resources := request(token, "org-core", resourcePath)
+ var listed struct {
+ Data []unifiedresources.Resource `json:"data"`
+ }
+ if resources.Code != 200 || json.Unmarshal(resources.Body.Bytes(), &listed) != nil {
+ t.Fatalf("resources %d: %s", resources.Code, resources.Body.String())
+ }
+ var host *unifiedresources.Resource
+ for j := range listed.Data {
+ if listed.Data[j].Type == unifiedresources.ResourceTypeAgent {
+ host = &listed.Data[j]
+ }
+ }
+ if host == nil || host.MetricsTarget == nil || host.MetricsTarget.ResourceID != "core-rest-fixture" || host.Agent == nil || host.Agent.Memory == nil {
+ t.Fatalf("canonical host/target lost: %s", resources.Body.String())
+ }
+ memory := host.Agent.Memory
+ knownCPU, knownMemory := i != 1, i != 1 && i != 2
+ if (host.Metrics.CPU != nil) != knownCPU || (host.Metrics.Memory != nil) != knownMemory || memory.UsageUnavailable == knownMemory {
+ t.Fatalf("unknown became idle: %+v %+v", host.Metrics, memory)
+ }
+ if memory.Total != 16<<30 || host.Metrics.Disk == nil || host.Metrics.Disk.Percent != 40 {
+ t.Fatal("capacity was lost or substituted for telemetry")
+ }
+ if knownMemory {
+ used, free, cache := int64(6<<30), int64(2<<30), int64(8<<30)
+ if i == 3 {
+ used, free, cache = 0, 16<<30, 0
+ }
+ if memory.Used != used || memory.Free != free || memory.Cache != cache {
+ t.Fatalf("free/ARC/effective used changed: %+v", memory)
+ }
+ }
+ history24 := request(token, "org-core", strings.Replace(historyPath, "range=1h", "range=24h", 1))
+ history := request(token, "org-core", historyPath)
+ wantCalls := int64(3)
+ if i == 1 {
+ wantCalls = 21
+ }
+ if fixture.Posts.Load()-beforePosts != wantCalls || fixture.RejectedOldKey.Load() != 0 {
+ t.Fatal("chart did not traverse native reporting, or repeated the rejected envelope")
+ }
+ if history24.Code != history.Code {
+ t.Fatal("default-range chart status disagrees with selected range")
+ }
+ if i == 1 && snapshot.System.Telemetry.ErrorCategory != "request_rejected" {
+ t.Fatal("reporting rejection lost its diagnostic category")
+ }
+ var data struct {
+ ResourceType, ResourceID string
+ Metrics map[string][]point
+ }
+ if history.Code != 200 || json.Unmarshal(history.Body.Bytes(), &data) != nil || data.ResourceType != "agent" || data.ResourceID != "core-rest-fixture" {
+ t.Fatalf("chart response %d: %s", history.Code, history.Body.String())
+ }
+ expected := map[string]float64{"cpu": 12, "memory": 37.5, "diskread": 1024, "diskwrite": 2048, "netin": 32, "netout": 64, "temperature": 42}
+ for metric, value := range expected {
+ known := knownMemory || (metric == "cpu" && knownCPU)
+ if !known {
+ if len(data.Metrics[metric]) != 0 {
+ t.Fatalf("cold chart fabricated %s: %+v", metric, data.Metrics[metric])
+ }
+ continue
+ }
+ if i == 3 && metric != "temperature" {
+ value = 0
+ }
+ points := data.Metrics[metric]
+ if len(points) != 4 {
+ t.Fatalf("native %s observations lost: %+v", metric, points)
+ }
+ for j, p := range points {
+ if p.Value != value || p.Min != value || p.Max != value || (j > 0 && p.Timestamp <= points[j-1].Timestamp) {
+ t.Fatalf("native sample/bounds/order changed for %s: %+v", metric, points)
+ }
+ }
+ if points[3].Timestamp-points[0].Timestamp < 57*60*1000 {
+ t.Fatal("hour chart became a current-value shortcut")
+ }
+ }
+ // Pool capacity is independent of reporting and remains valid even
+ // when every optional CPU/memory/IO graph is rejected.
+ if len(data.Metrics["disk"]) != 1 || data.Metrics["disk"][0].Value != 40 {
+ t.Fatal("reporting failure erased independent capacity history")
+ }
+
+ payload, err := json.Marshal(map[string]any{"phase": phase, "resources": json.RawMessage(resources.Body.Bytes()), "history": map[string]any{"status": history.Code, "body": json.RawMessage(history.Body.Bytes())}, "history24": map[string]any{"status": history24.Code, "body": json.RawMessage(history24.Body.Bytes())}, "telemetry": snapshot.System.Telemetry})
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Logf("CORE_CHART_BROWSER %s", payload)
+ if i == 0 {
+ retained := monitor.GetGuestMetrics("agent:core-rest-fixture", time.Hour)
+ before := fixture.Posts.Load()
+ fixture.Phase.Store(1)
+ if err := provider.Refresh(context.Background()); err != nil {
+ t.Fatal(err)
+ }
+ publish()
+ resources := request(token, "org-core", resourcePath)
+ var current struct{ Data []unifiedresources.Resource }
+ if resources.Code != 200 || json.Unmarshal(resources.Body.Bytes(), &current) != nil {
+ t.Fatal("warm resource read failed")
+ }
+ for _, row := range current.Data {
+ if row.Type == unifiedresources.ResourceTypeAgent && (row.Metrics.CPU != nil || row.Metrics.Memory != nil) {
+ t.Fatal("retained History became current telemetry")
+ }
+ }
+ history24 := request(token, "org-core", strings.Replace(historyPath, "range=1h", "range=24h", 1))
+ history := request(token, "org-core", historyPath)
+ var warm struct{ Metrics map[string][]point }
+ if history.Code != 200 || history24.Code != 200 || json.Unmarshal(history.Body.Bytes(), &warm) != nil || fixture.Posts.Load()-before != 21 {
+ t.Fatal("warm native failure discarded valid History or escaped its request bound")
+ }
+ if len(retained["cpu"]) != 1 || len(retained["memory"]) != 1 {
+ t.Fatal("warm control has no original CPU/memory observations")
+ }
+ for metric, points := range retained {
+ if metric == "disk" {
+ continue
+ } // capacity was collected again; it is not a retained optional reading
+ if len(points) == 0 {
+ if len(warm.Metrics[metric]) != 0 {
+ t.Fatalf("warm failure invented unobserved %s", metric)
+ }
+ continue
+ }
+
+ if len(points) != 1 || len(warm.Metrics[metric]) != 1 {
+ t.Fatalf("warm %s history changed cardinality", metric)
+ }
+ old, got := points[0], warm.Metrics[metric][0]
+ wantTime := (old.Timestamp.Unix()/15*15 + 7) * 1000
+ if got.Timestamp != wantTime || got.Value != old.Value || got.Min != old.Value || got.Max != old.Value {
+ t.Fatalf("warm %s changed observation time/value: old %+v, returned %+v", metric, old, got)
+ }
+ }
+ payload, err := json.Marshal(map[string]any{"phase": "retained", "resources": json.RawMessage(resources.Body.Bytes()), "history": map[string]any{"status": history.Code, "body": json.RawMessage(history.Body.Bytes())}, "history24": map[string]any{"status": history24.Code, "body": json.RawMessage(history24.Body.Bytes())}, "telemetry": provider.Snapshot().System.Telemetry})
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Logf("CORE_CHART_BROWSER %s", payload)
+ }
+
+ })
+ }
+ before := fixture.Posts.Load()
+ if cfg.RemoveAPIToken(readToken.ID) == nil {
+ t.Fatal("fixture token missing")
+ }
+ cfg.SortAPITokens()
+ if rec := request(token, "org-core", historyPath); rec.Code != 401 || fixture.Posts.Load() != before {
+ t.Fatal("revoked token still reached native History")
+ }
+}
diff --git a/internal/monitoring/canonical_guardrails_test.go b/internal/monitoring/canonical_guardrails_test.go
index 45c59322b4..7531c0d5e3 100644
--- a/internal/monitoring/canonical_guardrails_test.go
+++ b/internal/monitoring/canonical_guardrails_test.go
@@ -3473,3 +3473,34 @@ func TestVMwareInventoryRefreshBudgetContract(t *testing.T) {
t.Run("configuration", testVMwarePollTimeoutConfiguration)
t.Run("refresh-and-projection", testVMwarePollRefreshBudgetAndProjection)
}
+
+// The REST seed must preserve the same resolved coordinates as the broadcast
+// generation, without consulting a later registry or editing a shared view.
+func TestUnifiedSnapshotPreservesCapturedHistoryTargets(t *testing.T) {
+ oldTarget := &unifiedresources.MetricsTarget{ResourceType: "agent", ResourceID: "old-connection"}
+ rows := []unifiedresources.Resource{
+ {ID: "canonical-row", Type: unifiedresources.ResourceTypeAgent, MetricsTarget: oldTarget},
+ {ID: "unknown-row", Type: unifiedresources.ResourceTypeAgent, MetricsTarget: oldTarget},
+ }
+ targets := projectionMetricsTargets{"canonical-row": {ResourceType: "agent", ResourceID: "native-connection"}}
+ view := monitorUnifiedStateView{resources: rows, metricsTargets: targets}
+ got := unifiedSnapshotResources(view)
+ if got[0].MetricsTarget == nil || got[0].MetricsTarget.ResourceID != "native-connection" || got[1].MetricsTarget != nil {
+ t.Fatalf("captured known/unknown targets lost: %+v", got)
+ }
+ if rows[0].MetricsTarget != oldTarget || rows[1].MetricsTarget != oldTarget || oldTarget.ResourceID != "old-connection" {
+ t.Fatal("snapshot decorated the shared read-state rows")
+ }
+ got[0].MetricsTarget.ResourceID = "caller-change"
+ got[0].Name = "caller-change"
+ again := unifiedSnapshotResources(view)
+ if again[0].Name != "" || again[0].MetricsTarget.ResourceID != "native-connection" || targets["canonical-row"].ResourceID != "native-connection" {
+ t.Fatal("returned snapshot changed the captured generation")
+ }
+ // A store without a captured target projection retains its existing
+ // source row; the snapshot does not invent a coordinate from its name.
+ fallback := unifiedSnapshotResources(monitorUnifiedStateView{resources: rows})
+ if fallback[0].MetricsTarget != oldTarget {
+ t.Fatal("projection-less fallback lost its supplied coordinate")
+ }
+}
diff --git a/internal/monitoring/resource_stale_thresholds.go b/internal/monitoring/resource_stale_thresholds.go
index 7f51b37fe9..9a0d85f25d 100644
--- a/internal/monitoring/resource_stale_thresholds.go
+++ b/internal/monitoring/resource_stale_thresholds.go
@@ -140,7 +140,26 @@ func resourceStaleThresholdsForIntervals(
func (m *Monitor) UnifiedResourceSnapshotWithStaleThresholds() ([]unifiedresources.Resource, time.Time, map[unifiedresources.DataSource]time.Duration) {
m.refreshUnifiedResourceRead()
view := m.currentUnifiedStateView()
- return m.applyPersistedMetadataToUnifiedResources(view.resources), view.freshness, view.staleThresholds
+ return m.applyPersistedMetadataToUnifiedResources(unifiedSnapshotResources(view)), view.freshness, view.staleThresholds
+}
+
+// unifiedSnapshotResources carries the captured generation's resolved History
+// coordinates across the REST registry rebuild. Source-native IDs cannot always
+// be reconstructed from presentation rows (notably TrueNAS and VMware). The
+// targets already belong to this read, so never consult the live store again or
+// guess from a display name. Copy top-level rows before decorating: mock views
+// and cached read-state overlays may share their resource slice with readers.
+func unifiedSnapshotResources(view monitorUnifiedStateView) []unifiedresources.Resource {
+ if view.metricsTargets == nil {
+ return view.resources
+ }
+ resources := append([]unifiedresources.Resource(nil), view.resources...)
+ for i := range resources {
+ // Missing in this captured generation is authoritative too; do not
+ // carry a stale target from an earlier presentation row.
+ resources[i].MetricsTarget = view.metricsTargets.MetricsTargetForResource(resources[i].ID)
+ }
+ return resources
}
// Headless readers need the same publication freshness as frontend state reads.
diff --git a/internal/monitoring/truenas_poller_test.go b/internal/monitoring/truenas_poller_test.go
index eedb2c6eb4..7af12fa8f8 100644
--- a/internal/monitoring/truenas_poller_test.go
+++ b/internal/monitoring/truenas_poller_test.go
@@ -2544,7 +2544,7 @@ func TestTrueNASPartialReportingPipeline(t *testing.T) {
break
}
var query struct {
- Query map[string]any `json:"query"`
+ Query map[string]any `json:"reporting_query"`
Graphs []map[string]any `json:"graphs"`
}
if err := json.NewDecoder(r.Body).Decode(&query); err != nil {
diff --git a/internal/testutil/truenascore/rest.go b/internal/testutil/truenascore/rest.go
new file mode 100644
index 0000000000..e77f27373e
--- /dev/null
+++ b/internal/testutil/truenascore/rest.go
@@ -0,0 +1,131 @@
+// Package truenascore supplies the explicit CORE 13 REST schema used by
+// read-only client/resource/chart integration tests. It is not an appliance.
+package truenascore
+
+import (
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "sync/atomic"
+ "testing"
+)
+
+// RESTFixture rejects the reported old request key independently of the
+// production request type. Phase: complete, rejected, CPU-only, zero, recovered.
+// Set WideHistory before the first request to return four native observations
+// spanning the requested hour, rather than two live observations ten seconds apart.
+type RESTFixture struct {
+ Server *httptest.Server
+ Phase, Posts, CatalogueCalls, RejectedOldKey atomic.Int64
+ WideHistory bool
+}
+
+func NewRESTFixture(t *testing.T) *RESTFixture {
+ t.Helper()
+ f := &RESTFixture{}
+ f.Server = httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ switch r.URL.Path {
+ case "/api/v2.0/system/info":
+ fmt.Fprint(w, `{"hostname":"core-fixture","version":"TrueNAS-13.0-U6.8","cores":8,"physmem":17179869184}`)
+ case "/api/v2.0/pool":
+ fmt.Fprint(w, `[{"id":1,"name":"tank","status":"ONLINE","size":1000,"allocated":400,"free":600}]`)
+ case "/api/v2.0/disk":
+ fmt.Fprint(w, `[{"identifier":"disk-a","name":"da0","serial":"fixture-disk","rotationrate":"unknown","size":1000}]`)
+ case "/api/v2.0/pool/dataset", "/api/v2.0/alert/list":
+ fmt.Fprint(w, `[]`)
+ case "/api/v2.0/reporting/graphs":
+ f.CatalogueCalls.Add(1)
+ fmt.Fprint(w, `[{"name":"disk","identifiers":["da0"]},{"name":"interface","identifiers":["vtnet0"]}]`)
+ case "/api/v2.0/reporting/get_data":
+ f.Posts.Add(1)
+ var body map[string]json.RawMessage
+ if r.Method != http.MethodPost || json.NewDecoder(r.Body).Decode(&body) != nil {
+ http.Error(w, "invalid fixture request", http.StatusBadRequest)
+ return
+ }
+ if _, present := body["query"]; present {
+ f.RejectedOldKey.Add(1)
+ w.WriteHeader(http.StatusBadRequest)
+ fmt.Fprint(w, `{"message":"The following attributes are not expected: query"}`)
+ return
+ }
+ var graphs []struct{ Name, Identifier string }
+ var window struct {
+ Start, End int64
+ Aggregate bool
+ }
+ if len(body) != 2 || json.Unmarshal(body["graphs"], &graphs) != nil || json.Unmarshal(body["reporting_query"], &window) != nil || !window.Aggregate || window.Start >= window.End {
+ http.Error(w, "missing graphs/reporting_query range", http.StatusBadRequest)
+ return
+ }
+ if f.Phase.Load() == 1 {
+ w.WriteHeader(http.StatusBadRequest)
+ fmt.Fprint(w, `{"message":"synthetic reporting validation failure"}`)
+ return
+ }
+ var result []map[string]any
+ for _, graph := range graphs {
+ var legend []string
+ var values []any
+ zero := f.Phase.Load() == 3
+ if f.Phase.Load() == 2 && graph.Name != "cpu" {
+ continue // usable CPU does not establish any sibling reading
+ }
+ switch graph.Name {
+ case "cpu":
+ legend, values = []string{"user", "nice", "system", "interrupt", "idle"}, []any{8, 0, 3, 1, 88}
+ if zero {
+ values = []any{0, 0, 0, 0, 100}
+ }
+ case "memory":
+ legend = []string{"memory-active_value", "memory-inactive_value", "memory-wired_value", "memory-laundry_value", "memory-free_value"}
+ values = []any{1 << 30, 2 << 30, 11 << 30, 0, 2 << 30}
+ if zero {
+ values = []any{0, 0, 0, 0, 16 << 30}
+ }
+ case "arcsize":
+ legend, values = []string{"arcsize_value"}, []any{8 << 30}
+ if zero {
+ values = []any{0}
+ }
+ case "disk":
+ if graph.Identifier != "da0" {
+ t.Error("disk identifier lost")
+ }
+ legend, values = []string{"disk_octets_read", "disk_octets_write"}, []any{1024, 2048}
+ if zero {
+ values = []any{0, 0}
+ }
+ case "interface":
+ if graph.Identifier != "vtnet0" {
+ t.Error("interface identifier lost")
+ }
+ legend, values = []string{"rx", "tx", "overlap"}, []any{32, 64, 32}
+ if zero {
+ values = []any{0, 0, 0}
+ }
+ case "cputemp":
+ legend, values = []string{"temperature"}, []any{42}
+ default:
+ t.Errorf("unexpected graph %s", graph.Name)
+ }
+ start, step := window.End-10, int64(10)
+ data := []any{values, values}
+ if f.WideHistory && window.End-window.Start >= 3600 {
+ start, step = window.End-3540, 1160
+ data = []any{values, values, values, values}
+ }
+ result = append(result, map[string]any{"name": graph.Name, "identifier": graph.Identifier, "legend": legend, "start": start, "end": window.End, "step": step, "data": data})
+ }
+ if err := json.NewEncoder(w).Encode(result); err != nil {
+ t.Error(err)
+ }
+ default:
+ http.NotFound(w, r)
+ }
+ }))
+ t.Cleanup(f.Server.Close)
+ return f
+}
diff --git a/internal/truenas/client.go b/internal/truenas/client.go
index ec12697deb..47b98c3409 100644
--- a/internal/truenas/client.go
+++ b/internal/truenas/client.go
@@ -480,13 +480,14 @@ func isReportingGraphFailure(err error) bool {
}
// getReportingDataREST issues reporting.get_data over the REST v2.0 transport.
-// The middleware method takes positional parameters (graphs, query); REST
-// serves it as a POST body keyed by parameter name.
+// WebSocket parameters are positional, but REST v2.0 binds top-level keys to
+// the middleware schema names: graphs and reporting_query, not the Go/RPC
+// variable name query. CORE rejects an unknown query key before reading graphs.
func (c *Client) getReportingDataREST(ctx context.Context, graphs []map[string]any, query map[string]any) ([]trueNASReportingGetDataResponse, error) {
var response []trueNASReportingGetDataResponse
err := c.postJSON(ctx, "/reporting/get_data", map[string]any{
- "graphs": graphs,
- "query": query,
+ "graphs": graphs,
+ "reporting_query": query,
}, &response)
if err != nil {
return nil, err
diff --git a/internal/truenas/client_test.go b/internal/truenas/client_test.go
index e1f305cbc2..d2446ec2aa 100644
--- a/internal/truenas/client_test.go
+++ b/internal/truenas/client_test.go
@@ -2134,7 +2134,7 @@ func (s *isolatedReportingTransport) RoundTrip(r *http.Request) (*http.Response,
s.calls++
var request struct {
Graphs []map[string]any `json:"graphs"`
- Query map[string]any `json:"query"`
+ Query map[string]any `json:"reporting_query"`
}
if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
return nil, err
@@ -2300,7 +2300,7 @@ func (s *nativeShapeReportingTransport) RoundTrip(r *http.Request) (*http.Respon
}
var request struct {
Graphs []map[string]any `json:"graphs"`
- Query map[string]any `json:"query"`
+ Query map[string]any `json:"reporting_query"`
}
if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
return nil, err
diff --git a/internal/truenas/core_reporting_test.go b/internal/truenas/core_reporting_test.go
index 08a2594fde..6a6bddfffd 100644
--- a/internal/truenas/core_reporting_test.go
+++ b/internal/truenas/core_reporting_test.go
@@ -243,10 +243,15 @@ func (s *core13ReportingTransport) RoundTrip(r *http.Request) (*http.Response, e
s.reportingCalls++
var request struct {
Graphs []map[string]any `json:"graphs"`
- Query map[string]any `json:"query"`
+ Query map[string]any `json:"reporting_query"`
}
- if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
- return nil, err
+ decoder := json.NewDecoder(r.Body)
+ decoder.DisallowUnknownFields()
+ if err := decoder.Decode(&request); err != nil {
+ return nil, fmt.Errorf("invalid CORE REST reporting envelope: %w", err)
+ }
+ if len(request.Graphs) == 0 || request.Query["aggregate"] != true {
+ return nil, fmt.Errorf("missing graphs or reporting_query range")
}
s.requested = append(s.requested, request.Graphs...)
var responses []trueNASReportingGetDataResponse
diff --git a/internal/truenas/reporting_aggregation_test.go b/internal/truenas/reporting_aggregation_test.go
index 1fcecb8b7a..286ab6eec1 100644
--- a/internal/truenas/reporting_aggregation_test.go
+++ b/internal/truenas/reporting_aggregation_test.go
@@ -227,7 +227,7 @@ func TestRESTReportingDefaultAggregationsPreserveSamples(t *testing.T) {
calls++
var request struct {
Graphs []map[string]any `json:"graphs"`
- Query map[string]any `json:"query"`
+ Query map[string]any `json:"reporting_query"`
}
if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
return nil, err