Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Preserve fractional TrueNAS app stats across reused sessions
pulse-triage[bot]b052aedb0f5c
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex 8781dad42e..0da4e12c66 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -589,6 +589,35 @@ previously observed-but-failing runtime states. A synthetic successful loginwith a required pool-method failure is not attribution of the native #2382regression; the failing appliance/session method remains field evidence.+### TrueNAS app CPU JSON numbers preserve readings and sessions — issue #2400++`app.stats` CPU usage is a percentage-valued JSON number, not an integer+counter. Fractional values and idle `0.0` must decode without rejecting the+whole event; legacy integer values remain supported. Preserve the emitted+percentage without truncation or an additional ×100 conversion. Memory,+network and block-I/O integer fields retain their existing units and decoding.++Successful subscribe/read/unsubscribe must reuse the authenticated session+across GetApps and provider polls. A fresh event advances its collected-at+observation and the connection-scoped canonical CPU/memory/network readings+and History; rereading the same observation must not renew History. Malformed+CPU types and out-of-range JSON numbers still reject the complete event,+discard its unusable session and return no partial sample, without retrying a+non-transport decoding error. Existing permission/termination/deadline,+transport retry, cleanup, authentication, identity and tenant boundaries stay.+No session lifetime, keepalive, poll interval or freshness threshold changes.++`TestJSONRPCAppStatsNumbersReuseSession` and+`TestJSONRPCAppStatsRejectsMalformedCPUNumbers` in+`internal/truenas/transport_test.go` exercise actual GetApps subscriptions,+reused login, numeric compatibility and malformed-event disposal/recovery.+`TestTrueNASFractionalAppStatsPollAndHistory` in+`internal/monitoring/monitor_polling_test.go` connects TLS/RPC, thirteen-app+inventory, repeated actual provider polls, canonical resources and all four+CPU/memory/network History series with stable connection/tenant identity.+These short local polls are source regressions, not native SCALE/Traefik+sixty-second session acceptance, installed restart or containing publication.+### TrueNAS empty app inventories and session-disposal evidenceAn empty or null `app.query`/legacy `/app` result remains a successful emptydiff --git a/internal/monitoring/monitor_polling_test.go b/internal/monitoring/monitor_polling_test.goindex bde1dfa401..cf941bfe4b 100644--- a/internal/monitoring/monitor_polling_test.go+++ b/internal/monitoring/monitor_polling_test.go@@ -2,12 +2,20 @@ package monitoringimport ("context"+ "crypto/sha256"+ "encoding/json""errors"+ "fmt"+ "math"+ "net/http"+ "net/http/httptest"+ "strconv""strings""sync/atomic""testing""time"+ "github.com/gorilla/websocket""github.com/rcourtman/pulse-go-rewrite/internal/alerts""github.com/rcourtman/pulse-go-rewrite/internal/config""github.com/rcourtman/pulse-go-rewrite/internal/mock"@@ -534,6 +542,211 @@ func TestBuildBroadcastFrontendStateIncludesUnifiedIncidentAlerts(t *testing.T)}}+// #2400: exercise the actual TLS/RPC -> GetApps -> APIFetcher -> provider+// poll -> canonical resources -> History path, not only numeric unmarshalling.+// Three short, explicitly due polls are a source regression, not a native+// sixty-second/Traefik soak. Only the first two of thirteen apps' CPU shapes+// come from the reporter; names and all remaining inventory are synthetic.+func TestTrueNASFractionalAppStatsPollAndHistory(t *testing.T) {+ previous := truenas.IsFeatureEnabled()+ truenas.SetFeatureEnabled(true)+ t.Cleanup(func() { truenas.SetFeatureEnabled(previous) })+ const appCount = 13+ const hostMemory = int64(8 * 1024 * 1024 * 1024)+ cpus := [][2]string{{"0.0286", "0.0"}, {"0.0017857142857142859", "0.0"}, {"17", "0"}}+ var poll atomic.Int32+ var sessions, logins, statsSubscriptions, realtimeSubscriptions, unsubscribes atomic.Int32+ upgrader := websocket.Upgrader{}+ server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.URL.Path != "/api/current" || !websocket.IsWebSocketUpgrade(r) {+ t.Error("modern polling unexpectedly requested REST")+ http.NotFound(w, r)+ return+ }+ conn, err := upgrader.Upgrade(w, r, nil)+ if err != nil {+ t.Error(err)+ return+ }+ defer conn.Close()+ sessions.Add(1)+ for {+ var request struct {+ ID int64 `json:"id"`+ Method string `json:"method"`+ Params []json.RawMessage `json:"params"`+ }+ if err := conn.ReadJSON(&request); err != nil {+ return+ }+ currentPoll := int(poll.Load())+ var result any = []any{}+ var collection string+ var fields any+ switch request.Method {+ case "auth.login_ex":+ logins.Add(1)+ result = map[string]any{"response_type": "SUCCESS"}+ case "system.info":+ result = map[string]any{"hostname": "fractional-nas", "version": "TrueNAS-SCALE-26.0.0-BETA.3", "system_serial": "FIXTURE", "physical_cores": 4, "physmem": hostMemory}+ case "app.query":+ apps := make([]any, appCount)+ for i := range apps {+ name := fmt.Sprintf("fixture-app-%d", i)+ apps[i] = map[string]any{"id": name, "name": name, "state": "RUNNING"}+ }+ result = apps+ case "core.subscribe":+ if len(request.Params) != 1 || json.Unmarshal(request.Params[0], &collection) != nil {+ t.Error("invalid subscription arguments")+ return+ }+ switch collection {+ case `reporting.realtime:{"interval":2}`:+ realtimeSubscriptions.Add(1)+ result = "realtime-sub"+ fields = map[string]any{"cpu": map[string]any{"usage": 10}, "memory": map[string]any{"physical_memory_total": hostMemory, "physical_memory_available": hostMemory / 2}}+ case `app.stats:{"interval":2}`:+ statsSubscriptions.Add(1)+ result = "app-sub"+ apps := make([]any, appCount)+ for i := range apps {+ cpu := strconv.Itoa(i)+ if i < 2 {+ cpu = cpus[currentPoll][i]+ }+ apps[i] = map[string]any{+ "app_name": fmt.Sprintf("fixture-app-%d", i), "cpu_usage": json.Number(cpu),+ "memory": 1072459776 + currentPoll*1024 + i,+ "blkio": map[string]any{"read": 29720576 + currentPoll*4096, "write": currentPoll * 2048},+ "networks": []any{+ map[string]any{"interface_name": "eth0", "rx_bytes": 3671 + currentPoll, "tx_bytes": 284 + currentPoll},+ map[string]any{"interface_name": "eth1", "rx_bytes": 2, "tx_bytes": 1},+ },+ }+ }+ fields = apps+ default:+ t.Errorf("unexpected subscription %s", collection)+ return+ }+ case "core.unsubscribe":+ var id string+ if len(request.Params) != 1 || json.Unmarshal(request.Params[0], &id) != nil || (id != "app-sub" && id != "realtime-sub") {+ t.Error("subscription cleanup used an unexpected ID")+ return+ }+ unsubscribes.Add(1)+ result = nil+ case "reporting.get_data", "pool.query", "boot.get_state", "pool.dataset.query", "disk.query", "alert.list", "service.query", "vm.query", "sharing.smb.query", "sharing.nfs.query", "zfs.resource.snapshot.query", "replication.query":+ default:+ t.Errorf("unexpected or mutating RPC method %s", request.Method)+ return+ }+ if conn.WriteJSON(map[string]any{"jsonrpc": "2.0", "id": request.ID, "result": result}) != nil {+ return+ }+ if collection != "" {+ if conn.WriteJSON(map[string]any{"jsonrpc": "2.0", "method": "collection_update", "params": map[string]any{"collection": collection, "fields": fields}}) != nil {+ return+ }+ }+ }+ }))+ t.Cleanup(server.Close)+ client, err := truenas.NewClient(truenas.ClientConfig{+ Host: server.URL, APIKey: "fixture-key", Username: "fixture-user", Timeout: 2 * time.Second,+ Fingerprint: fmt.Sprintf("%x", sha256.Sum256(server.Certificate().Raw)),+ })+ if err != nil {+ t.Fatal(err)+ }+ t.Cleanup(client.Close)+ instance := config.TrueNASInstance{ID: "fractional-connection", Host: server.URL, Enabled: true, PollIntervalSecs: 60}+ provider := truenas.NewLiveProviderForConnection(&truenas.APIFetcher{Client: client}, instance.ID)+ poller := NewTrueNASPoller(nil, 0, nil)+ poller.providersByOrg["default"] = map[string]*truenas.Provider{instance.ID: provider}+ poller.configsByOrg["default"] = map[string]config.TrueNASInstance{instance.ID: instance}+ resourceStore := unifiedresources.NewMonitorAdapter(nil)+ monitor := &Monitor{resourceStore: resourceStore, metricsHistory: NewMetricsHistory(1024, 24*time.Hour)}+ var lastSuccess, lastStats time.Time+ var firstIDs = make(map[string]string)+ for currentPoll := range cpus {+ poll.Store(int32(currentPoll))+ poller.ensureConnectionRuntimeStatusLocked("default", instance.ID).nextPollAt = time.Now().Add(-time.Second)+ poller.pollAll(context.Background())+ summary := poller.ConnectionSummaries("default", []config.TrueNASInstance{instance})[instance.ID]+ if summary.Poll == nil || summary.Poll.LastSuccessAt == nil || summary.Poll.LastError != nil || !summary.Poll.LastSuccessAt.After(lastSuccess) {+ t.Fatalf("poll %d did not complete with fresh success: %+v", currentPoll, summary)+ }+ lastSuccess = *summary.Poll.LastSuccessAt+ if summary.Transport == nil || summary.Transport.AuthMechanism != "auth.login_ex" || !summary.Transport.Connected || summary.Transport.Reconnects != 0 {+ t.Fatalf("fractional stats discarded/re-authenticated the session: %+v", summary.Transport)+ }+ snapshot := provider.Snapshot()+ if snapshot == nil || len(snapshot.Apps) != appCount || snapshot.System.CPUPercent != 10 {+ t.Fatalf("poll %d lost inventory or host telemetry", currentPoll)+ }+ if snapshot.Apps[0].Stats == nil || !snapshot.Apps[0].Stats.CollectedAt.After(lastStats) {+ t.Fatalf("poll %d did not collect fresh app stats", currentPoll)+ }+ lastStats = snapshot.Apps[0].Stats.CollectedAt+ resourceStore.PopulateSnapshotAndSupplemental(models.StateSnapshot{}, map[unifiedresources.DataSource][]unifiedresources.IngestRecord{+ unifiedresources.SourceTrueNAS: poller.GetCurrentRecordsForOrg("default"),+ })+ monitor.syncUnifiedAppContainerMetrics(resourceStore)+ appResources := 0+ for _, resource := range resourceStore.GetAll() {+ if resource.Type != unifiedresources.ResourceTypeAppContainer {+ continue+ }+ appResources+++ var i int+ if _, err := fmt.Sscanf(resource.Name, "fixture-app-%d", &i); err != nil || i < 0 || i >= appCount {+ t.Fatalf("unexpected app identity %s", resource.Name)+ }+ cpu := float64(i)+ if i < 2 {+ cpu, _ = strconv.ParseFloat(cpus[currentPoll][i], 64)+ }+ memory := int64(1072459776 + currentPoll*1024 + i)+ want := map[string]float64{"cpu": cpu, "memory": float64(memory) / float64(hostMemory) * 100, "netin": float64(3673 + currentPoll), "netout": float64(285 + currentPoll)}+ if resource.Metrics == nil || resource.Metrics.CPU == nil || resource.Metrics.CPU.Percent != cpu ||+ resource.Metrics.Memory == nil || resource.Metrics.Memory.Used == nil || *resource.Metrics.Memory.Used != memory ||+ resource.Metrics.NetIn == nil || resource.Metrics.NetIn.Value != want["netin"] || resource.Metrics.NetOut == nil || resource.Metrics.NetOut.Value != want["netout"] ||+ resource.TrueNAS == nil || resource.TrueNAS.App == nil || resource.TrueNAS.App.Stats == nil || !resource.TrueNAS.App.Stats.CollectedAt.Equal(lastStats) {+ t.Fatalf("poll %d app %d lost precision, sibling readings or observation time: %+v", currentPoll, i, resource)+ }+ if currentPoll == 0 {+ firstIDs[resource.Name] = resource.ID+ } else if resource.ID != firstIDs[resource.Name] {+ t.Fatalf("poll %d changed canonical app identity", currentPoll)+ }+ target := resourceStore.MetricsTargetForResource(resource.ID)+ if target == nil || target.ResourceType != "app-container" || target.ResourceID != "system:fractional-connection/app:"+resource.Name {+ t.Fatalf("unexpected connection-scoped History target: %+v", target)+ }+ history := monitor.GetGuestMetrics("docker:"+target.ResourceID, time.Hour)+ for metric, value := range want {+ points := history[metric]+ if len(points) != currentPoll+1 || math.Abs(points[len(points)-1].Value-value) > 1e-12 || !points[len(points)-1].Timestamp.Equal(resource.LastSeen) {+ t.Fatalf("poll %d app %d %s History lost value/freshness: %+v, want %v", currentPoll, i, metric, points, value)+ }+ }+ }+ if appResources != appCount || len(poller.GetCurrentRecordsForOrg("other-tenant")) != 0 || sessions.Load() != 1 || logins.Load() != 1 ||+ statsSubscriptions.Load() != int32(currentPoll+1) || realtimeSubscriptions.Load() != int32(currentPoll+1) || unsubscribes.Load() != int32(2*(currentPoll+1)) {+ t.Fatalf("poll %d crossed tenant scope, lost apps or churned session/subscriptions: apps=%d sessions=%d logins=%d stats=%d realtime=%d unsubscribe=%d", currentPoll, appResources, sessions.Load(), logins.Load(), statsSubscriptions.Load(), realtimeSubscriptions.Load(), unsubscribes.Load())+ }+ // Rendering/History without a new observation must not renew samples.+ monitor.syncUnifiedAppContainerMetrics(resourceStore)+ }+ client.Close()+ if client.TransportStatus().Connected {+ t.Fatal("closing the client left its authenticated session active")+ }+}+func TestSyncUnifiedAppContainerMetricsRecordsTrueNASHistory(t *testing.T) {previous := truenas.IsFeatureEnabled()truenas.SetFeatureEnabled(true)diff --git a/internal/truenas/client.go b/internal/truenas/client.goindex 7e2d59cc7a..38817f03db 100644--- a/internal/truenas/client.go+++ b/internal/truenas/client.go@@ -2400,8 +2400,10 @@ type trueNASCollectionUpdate struct {}type trueNASAppStatsEvent struct {- AppName string `json:"app_name"`- CPUUsage int64 `json:"cpu_usage"`+ AppName string `json:"app_name"`+ // SCALE reports a percentage as a JSON number, including fractional and+ // idle 0.0 values. Integer decoding rejects the whole event and session.+ CPUUsage float64 `json:"cpu_usage"`Memory int64 `json:"memory"`Networks []trueNASAppNetworkStats `json:"networks"`BlkIO trueNASAppBlkIOStats `json:"blkio"`@@ -2740,7 +2742,7 @@ func (c *trueNASRPCClient) readAppStatsEvent(ctx context.Context, intervalSecond}appStats := AppStats{- CPUPercent: float64(field.CPUUsage),+ CPUPercent: field.CPUUsage,MemoryBytes: field.Memory,BlockReadBytes: field.BlkIO.Read,BlockWriteBytes: field.BlkIO.Write,diff --git a/internal/truenas/transport_test.go b/internal/truenas/transport_test.goindex 4dc03527f5..3b5c0b5b61 100644--- a/internal/truenas/transport_test.go+++ b/internal/truenas/transport_test.go@@ -987,6 +987,118 @@ func TestJSONRPCFetchSnapshotUsesPublishedTrueNAS26MethodsWithoutREST(t *testing}}+func TestJSONRPCAppStatsNumbersReuseSession(t *testing.T) {+ for _, cpu := range []string{"0.0286", "0.0", "17"} {+ t.Run(cpu, func(t *testing.T) {+ var logins, subscribes, unsubscribes atomic.Int32+ fixture := newProtocolFixture(t, func(_ int, request trueNASRPCRequest) protocolFixtureReply {+ switch request.Method {+ case "auth.login_ex":+ logins.Add(1)+ return protocolFixtureReply{result: map[string]any{"response_type": "SUCCESS"}}+ case "app.query":+ return protocolFixtureReply{result: []any{map[string]any{"id": "fixture-app", "name": "Fixture app", "state": "RUNNING"}}}+ case "core.subscribe":+ subscribes.Add(1)+ collection := request.Params.([]any)[0].(string)+ return protocolFixtureReply{result: "stats-sub", notifications: []protocolFixtureNotification{{+ method: "collection_update", params: map[string]any{+ "collection": collection, "fields": []any{map[string]any{+ "app_name": "fixture-app", "cpu_usage": json.Number(cpu), "memory": 1072459776,+ "networks": []any{map[string]any{"interface_name": "eth0", "rx_bytes": 3671, "tx_bytes": 284}},+ "blkio": map[string]any{"read": 29720576, "write": 0},+ }},+ },+ }}}+ case "core.unsubscribe":+ unsubscribes.Add(1)+ return protocolFixtureReply{result: nil}+ case "pool.query":+ return protocolFixtureReply{result: []any{}}+ default:+ t.Errorf("unexpected method %s", request.Method)+ return protocolFixtureReply{close: true}+ }+ }, nil)+ client := protocolFixtureClient(t, fixture.server.URL, ClientConfig{APIKey: "fixture-key", Username: "fixture-user"})+ wantCPU, _ := strconv.ParseFloat(cpu, 64)+ var lastStats time.Time+ for poll := 0; poll < 3; poll++ {+ apps, err := client.GetApps(context.Background())+ if err != nil || len(apps) != 1 || apps[0].ID != "fixture-app" {+ t.Fatalf("poll %d lost app inventory: %+v, %v", poll, apps, err)+ }+ if stats := apps[0].Stats; stats == nil {+ t.Errorf("poll %d lost app.stats for JSON CPU number %s", poll, cpu)+ } else {+ if stats.CPUPercent != wantCPU || stats.MemoryBytes != 1072459776 || stats.NetInRate != 3671 || stats.NetOutRate != 284 ||+ stats.BlockReadBytes != 29720576 || stats.BlockWriteBytes != 0 || stats.IntervalSeconds != 2 || !stats.CollectedAt.After(lastStats) {+ t.Errorf("poll %d lost precise CPU/sibling fields/freshness: %+v", poll, stats)+ }+ lastStats = stats.CollectedAt+ }+ }+ if _, err := client.GetPools(context.Background()); err != nil {+ t.Fatal(err)+ }+ if fixture.sessions.Load() != 1 || logins.Load() != 1 || subscribes.Load() != 3 || unsubscribes.Load() != 3 ||+ fixture.restRequests.Load() != 0 || !client.TransportStatus().Connected {+ t.Errorf("JSON CPU number caused session churn: sessions=%d logins=%d subscribe=%d unsubscribe=%d", fixture.sessions.Load(), logins.Load(), subscribes.Load(), unsubscribes.Load())+ }+ })+ }+}++// Invalid CPU data must still fail the complete event, not fabricate an idle+// sample or retry a non-transport decode failure after accepting JSON numbers.+func TestJSONRPCAppStatsRejectsMalformedCPUNumbers(t *testing.T) {+ for _, cpu := range []string{`"0.0286"`, `true`, `{}`, `[]`, `1e309`} {+ t.Run(cpu, func(t *testing.T) {+ var subscribes, unsubscribes, logins atomic.Int32+ fixture := newProtocolFixture(t, func(_ int, request trueNASRPCRequest) protocolFixtureReply {+ switch request.Method {+ case "auth.login_ex":+ logins.Add(1)+ return protocolFixtureReply{result: map[string]any{"response_type": "SUCCESS"}}+ case "core.subscribe":+ subscribes.Add(1)+ collection := request.Params.([]any)[0].(string)+ return protocolFixtureReply{result: "stats-sub", notifications: []protocolFixtureNotification{{+ method: "collection_update", params: map[string]any{+ "collection": collection,+ "fields": []any{+ map[string]any{"app_name": "valid-app", "cpu_usage": json.Number("0.0286")},+ map[string]any{"app_name": "invalid-app", "cpu_usage": json.RawMessage(cpu)},+ },+ },+ }}}+ case "core.unsubscribe":+ unsubscribes.Add(1)+ return protocolFixtureReply{result: nil}+ case "pool.query":+ return protocolFixtureReply{result: []any{}}+ default:+ t.Errorf("unexpected method %s", request.Method)+ return protocolFixtureReply{close: true}+ }+ }, nil)+ client := protocolFixtureClient(t, fixture.server.URL, ClientConfig{APIKey: "fixture-key", Username: "fixture-user"})+ stats, err := client.GetAppStats(context.Background())+ if err == nil || !strings.Contains(err.Error(), "decode truenas app.stats notification") || stats != nil {+ t.Fatalf("malformed CPU produced partial/successful stats: %+v, %v", stats, err)+ }+ if client.rpc != nil || client.TransportStatus().Connected || fixture.sessions.Load() != 1 ||+ logins.Load() != 1 || subscribes.Load() != 1 || unsubscribes.Load() != 0 {+ t.Fatal("malformed stats were retried, retained the failed session or attempted unsubscribe")+ }+ if _, err := client.GetPools(context.Background()); err != nil || fixture.sessions.Load() != 2 ||+ logins.Load() != 2 || fixture.restRequests.Load() != 0 {+ t.Fatalf("next independent read did not recover over authenticated JSON-RPC: %v", err)+ }+ })+ }+}+func TestJSONRPCMalformedSubscriptionEventDiscardsSession(t *testing.T) {fixture := newProtocolFixture(t, func(session int, request trueNASRPCRequest) protocolFixtureReply {switch request.Method {