Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Preserve fractional TrueNAS app stats across reused sessions

pulse-triage[bot]b052aedb0f5c
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index 8781dad42e..0da4e12c66 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -589,6 +589,35 @@ previously observed-but-failing runtime states. A synthetic successful login
with a required pool-method failure is not attribution of the native #2382
regression; the failing appliance/session method remains field evidence.
+### TrueNAS app CPU JSON numbers preserve readings and sessions — issue #2400
+
+`app.stats` CPU usage is a percentage-valued JSON number, not an integer
+counter. Fractional values and idle `0.0` must decode without rejecting the
+whole event; legacy integer values remain supported. Preserve the emitted
+percentage without truncation or an additional ×100 conversion. Memory,
+network and block-I/O integer fields retain their existing units and decoding.
+
+Successful subscribe/read/unsubscribe must reuse the authenticated session
+across GetApps and provider polls. A fresh event advances its collected-at
+observation and the connection-scoped canonical CPU/memory/network readings
+and History; rereading the same observation must not renew History. Malformed
+CPU types and out-of-range JSON numbers still reject the complete event,
+discard its unusable session and return no partial sample, without retrying a
+non-transport decoding error. Existing permission/termination/deadline,
+transport retry, cleanup, authentication, identity and tenant boundaries stay.
+No session lifetime, keepalive, poll interval or freshness threshold changes.
+
+`TestJSONRPCAppStatsNumbersReuseSession` and
+`TestJSONRPCAppStatsRejectsMalformedCPUNumbers` in
+`internal/truenas/transport_test.go` exercise actual GetApps subscriptions,
+reused login, numeric compatibility and malformed-event disposal/recovery.
+`TestTrueNASFractionalAppStatsPollAndHistory` in
+`internal/monitoring/monitor_polling_test.go` connects TLS/RPC, thirteen-app
+inventory, repeated actual provider polls, canonical resources and all four
+CPU/memory/network History series with stable connection/tenant identity.
+These short local polls are source regressions, not native SCALE/Traefik
+sixty-second session acceptance, installed restart or containing publication.
+
### TrueNAS empty app inventories and session-disposal evidence
An empty or null `app.query`/legacy `/app` result remains a successful empty
diff --git a/internal/monitoring/monitor_polling_test.go b/internal/monitoring/monitor_polling_test.go
index bde1dfa401..cf941bfe4b 100644
--- a/internal/monitoring/monitor_polling_test.go
+++ b/internal/monitoring/monitor_polling_test.go
@@ -2,12 +2,20 @@ package monitoring
import (
"context"
+ "crypto/sha256"
+ "encoding/json"
"errors"
+ "fmt"
+ "math"
+ "net/http"
+ "net/http/httptest"
+ "strconv"
"strings"
"sync/atomic"
"testing"
"time"
+ "github.com/gorilla/websocket"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
"github.com/rcourtman/pulse-go-rewrite/internal/config"
"github.com/rcourtman/pulse-go-rewrite/internal/mock"
@@ -534,6 +542,211 @@ func TestBuildBroadcastFrontendStateIncludesUnifiedIncidentAlerts(t *testing.T)
}
}
+// #2400: exercise the actual TLS/RPC -> GetApps -> APIFetcher -> provider
+// poll -> canonical resources -> History path, not only numeric unmarshalling.
+// Three short, explicitly due polls are a source regression, not a native
+// sixty-second/Traefik soak. Only the first two of thirteen apps' CPU shapes
+// come from the reporter; names and all remaining inventory are synthetic.
+func TestTrueNASFractionalAppStatsPollAndHistory(t *testing.T) {
+ previous := truenas.IsFeatureEnabled()
+ truenas.SetFeatureEnabled(true)
+ t.Cleanup(func() { truenas.SetFeatureEnabled(previous) })
+ const appCount = 13
+ const hostMemory = int64(8 * 1024 * 1024 * 1024)
+ cpus := [][2]string{{"0.0286", "0.0"}, {"0.0017857142857142859", "0.0"}, {"17", "0"}}
+ var poll atomic.Int32
+ var sessions, logins, statsSubscriptions, realtimeSubscriptions, unsubscribes atomic.Int32
+ upgrader := websocket.Upgrader{}
+ server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != "/api/current" || !websocket.IsWebSocketUpgrade(r) {
+ t.Error("modern polling unexpectedly requested REST")
+ http.NotFound(w, r)
+ return
+ }
+ conn, err := upgrader.Upgrade(w, r, nil)
+ if err != nil {
+ t.Error(err)
+ return
+ }
+ defer conn.Close()
+ sessions.Add(1)
+ for {
+ var request struct {
+ ID int64 `json:"id"`
+ Method string `json:"method"`
+ Params []json.RawMessage `json:"params"`
+ }
+ if err := conn.ReadJSON(&request); err != nil {
+ return
+ }
+ currentPoll := int(poll.Load())
+ var result any = []any{}
+ var collection string
+ var fields any
+ switch request.Method {
+ case "auth.login_ex":
+ logins.Add(1)
+ result = map[string]any{"response_type": "SUCCESS"}
+ case "system.info":
+ result = map[string]any{"hostname": "fractional-nas", "version": "TrueNAS-SCALE-26.0.0-BETA.3", "system_serial": "FIXTURE", "physical_cores": 4, "physmem": hostMemory}
+ case "app.query":
+ apps := make([]any, appCount)
+ for i := range apps {
+ name := fmt.Sprintf("fixture-app-%d", i)
+ apps[i] = map[string]any{"id": name, "name": name, "state": "RUNNING"}
+ }
+ result = apps
+ case "core.subscribe":
+ if len(request.Params) != 1 || json.Unmarshal(request.Params[0], &collection) != nil {
+ t.Error("invalid subscription arguments")
+ return
+ }
+ switch collection {
+ case `reporting.realtime:{"interval":2}`:
+ realtimeSubscriptions.Add(1)
+ result = "realtime-sub"
+ fields = map[string]any{"cpu": map[string]any{"usage": 10}, "memory": map[string]any{"physical_memory_total": hostMemory, "physical_memory_available": hostMemory / 2}}
+ case `app.stats:{"interval":2}`:
+ statsSubscriptions.Add(1)
+ result = "app-sub"
+ apps := make([]any, appCount)
+ for i := range apps {
+ cpu := strconv.Itoa(i)
+ if i < 2 {
+ cpu = cpus[currentPoll][i]
+ }
+ apps[i] = map[string]any{
+ "app_name": fmt.Sprintf("fixture-app-%d", i), "cpu_usage": json.Number(cpu),
+ "memory": 1072459776 + currentPoll*1024 + i,
+ "blkio": map[string]any{"read": 29720576 + currentPoll*4096, "write": currentPoll * 2048},
+ "networks": []any{
+ map[string]any{"interface_name": "eth0", "rx_bytes": 3671 + currentPoll, "tx_bytes": 284 + currentPoll},
+ map[string]any{"interface_name": "eth1", "rx_bytes": 2, "tx_bytes": 1},
+ },
+ }
+ }
+ fields = apps
+ default:
+ t.Errorf("unexpected subscription %s", collection)
+ return
+ }
+ case "core.unsubscribe":
+ var id string
+ if len(request.Params) != 1 || json.Unmarshal(request.Params[0], &id) != nil || (id != "app-sub" && id != "realtime-sub") {
+ t.Error("subscription cleanup used an unexpected ID")
+ return
+ }
+ unsubscribes.Add(1)
+ result = nil
+ case "reporting.get_data", "pool.query", "boot.get_state", "pool.dataset.query", "disk.query", "alert.list", "service.query", "vm.query", "sharing.smb.query", "sharing.nfs.query", "zfs.resource.snapshot.query", "replication.query":
+ default:
+ t.Errorf("unexpected or mutating RPC method %s", request.Method)
+ return
+ }
+ if conn.WriteJSON(map[string]any{"jsonrpc": "2.0", "id": request.ID, "result": result}) != nil {
+ return
+ }
+ if collection != "" {
+ if conn.WriteJSON(map[string]any{"jsonrpc": "2.0", "method": "collection_update", "params": map[string]any{"collection": collection, "fields": fields}}) != nil {
+ return
+ }
+ }
+ }
+ }))
+ t.Cleanup(server.Close)
+ client, err := truenas.NewClient(truenas.ClientConfig{
+ Host: server.URL, APIKey: "fixture-key", Username: "fixture-user", Timeout: 2 * time.Second,
+ Fingerprint: fmt.Sprintf("%x", sha256.Sum256(server.Certificate().Raw)),
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(client.Close)
+ instance := config.TrueNASInstance{ID: "fractional-connection", Host: server.URL, Enabled: true, PollIntervalSecs: 60}
+ provider := truenas.NewLiveProviderForConnection(&truenas.APIFetcher{Client: client}, instance.ID)
+ poller := NewTrueNASPoller(nil, 0, nil)
+ poller.providersByOrg["default"] = map[string]*truenas.Provider{instance.ID: provider}
+ poller.configsByOrg["default"] = map[string]config.TrueNASInstance{instance.ID: instance}
+ resourceStore := unifiedresources.NewMonitorAdapter(nil)
+ monitor := &Monitor{resourceStore: resourceStore, metricsHistory: NewMetricsHistory(1024, 24*time.Hour)}
+ var lastSuccess, lastStats time.Time
+ var firstIDs = make(map[string]string)
+ for currentPoll := range cpus {
+ poll.Store(int32(currentPoll))
+ poller.ensureConnectionRuntimeStatusLocked("default", instance.ID).nextPollAt = time.Now().Add(-time.Second)
+ poller.pollAll(context.Background())
+ summary := poller.ConnectionSummaries("default", []config.TrueNASInstance{instance})[instance.ID]
+ if summary.Poll == nil || summary.Poll.LastSuccessAt == nil || summary.Poll.LastError != nil || !summary.Poll.LastSuccessAt.After(lastSuccess) {
+ t.Fatalf("poll %d did not complete with fresh success: %+v", currentPoll, summary)
+ }
+ lastSuccess = *summary.Poll.LastSuccessAt
+ if summary.Transport == nil || summary.Transport.AuthMechanism != "auth.login_ex" || !summary.Transport.Connected || summary.Transport.Reconnects != 0 {
+ t.Fatalf("fractional stats discarded/re-authenticated the session: %+v", summary.Transport)
+ }
+ snapshot := provider.Snapshot()
+ if snapshot == nil || len(snapshot.Apps) != appCount || snapshot.System.CPUPercent != 10 {
+ t.Fatalf("poll %d lost inventory or host telemetry", currentPoll)
+ }
+ if snapshot.Apps[0].Stats == nil || !snapshot.Apps[0].Stats.CollectedAt.After(lastStats) {
+ t.Fatalf("poll %d did not collect fresh app stats", currentPoll)
+ }
+ lastStats = snapshot.Apps[0].Stats.CollectedAt
+ resourceStore.PopulateSnapshotAndSupplemental(models.StateSnapshot{}, map[unifiedresources.DataSource][]unifiedresources.IngestRecord{
+ unifiedresources.SourceTrueNAS: poller.GetCurrentRecordsForOrg("default"),
+ })
+ monitor.syncUnifiedAppContainerMetrics(resourceStore)
+ appResources := 0
+ for _, resource := range resourceStore.GetAll() {
+ if resource.Type != unifiedresources.ResourceTypeAppContainer {
+ continue
+ }
+ appResources++
+ var i int
+ if _, err := fmt.Sscanf(resource.Name, "fixture-app-%d", &i); err != nil || i < 0 || i >= appCount {
+ t.Fatalf("unexpected app identity %s", resource.Name)
+ }
+ cpu := float64(i)
+ if i < 2 {
+ cpu, _ = strconv.ParseFloat(cpus[currentPoll][i], 64)
+ }
+ memory := int64(1072459776 + currentPoll*1024 + i)
+ want := map[string]float64{"cpu": cpu, "memory": float64(memory) / float64(hostMemory) * 100, "netin": float64(3673 + currentPoll), "netout": float64(285 + currentPoll)}
+ if resource.Metrics == nil || resource.Metrics.CPU == nil || resource.Metrics.CPU.Percent != cpu ||
+ resource.Metrics.Memory == nil || resource.Metrics.Memory.Used == nil || *resource.Metrics.Memory.Used != memory ||
+ resource.Metrics.NetIn == nil || resource.Metrics.NetIn.Value != want["netin"] || resource.Metrics.NetOut == nil || resource.Metrics.NetOut.Value != want["netout"] ||
+ resource.TrueNAS == nil || resource.TrueNAS.App == nil || resource.TrueNAS.App.Stats == nil || !resource.TrueNAS.App.Stats.CollectedAt.Equal(lastStats) {
+ t.Fatalf("poll %d app %d lost precision, sibling readings or observation time: %+v", currentPoll, i, resource)
+ }
+ if currentPoll == 0 {
+ firstIDs[resource.Name] = resource.ID
+ } else if resource.ID != firstIDs[resource.Name] {
+ t.Fatalf("poll %d changed canonical app identity", currentPoll)
+ }
+ target := resourceStore.MetricsTargetForResource(resource.ID)
+ if target == nil || target.ResourceType != "app-container" || target.ResourceID != "system:fractional-connection/app:"+resource.Name {
+ t.Fatalf("unexpected connection-scoped History target: %+v", target)
+ }
+ history := monitor.GetGuestMetrics("docker:"+target.ResourceID, time.Hour)
+ for metric, value := range want {
+ points := history[metric]
+ if len(points) != currentPoll+1 || math.Abs(points[len(points)-1].Value-value) > 1e-12 || !points[len(points)-1].Timestamp.Equal(resource.LastSeen) {
+ t.Fatalf("poll %d app %d %s History lost value/freshness: %+v, want %v", currentPoll, i, metric, points, value)
+ }
+ }
+ }
+ if appResources != appCount || len(poller.GetCurrentRecordsForOrg("other-tenant")) != 0 || sessions.Load() != 1 || logins.Load() != 1 ||
+ statsSubscriptions.Load() != int32(currentPoll+1) || realtimeSubscriptions.Load() != int32(currentPoll+1) || unsubscribes.Load() != int32(2*(currentPoll+1)) {
+ t.Fatalf("poll %d crossed tenant scope, lost apps or churned session/subscriptions: apps=%d sessions=%d logins=%d stats=%d realtime=%d unsubscribe=%d", currentPoll, appResources, sessions.Load(), logins.Load(), statsSubscriptions.Load(), realtimeSubscriptions.Load(), unsubscribes.Load())
+ }
+ // Rendering/History without a new observation must not renew samples.
+ monitor.syncUnifiedAppContainerMetrics(resourceStore)
+ }
+ client.Close()
+ if client.TransportStatus().Connected {
+ t.Fatal("closing the client left its authenticated session active")
+ }
+}
+
func TestSyncUnifiedAppContainerMetricsRecordsTrueNASHistory(t *testing.T) {
previous := truenas.IsFeatureEnabled()
truenas.SetFeatureEnabled(true)
diff --git a/internal/truenas/client.go b/internal/truenas/client.go
index 7e2d59cc7a..38817f03db 100644
--- a/internal/truenas/client.go
+++ b/internal/truenas/client.go
@@ -2400,8 +2400,10 @@ type trueNASCollectionUpdate struct {
}
type trueNASAppStatsEvent struct {
- AppName string `json:"app_name"`
- CPUUsage int64 `json:"cpu_usage"`
+ AppName string `json:"app_name"`
+ // SCALE reports a percentage as a JSON number, including fractional and
+ // idle 0.0 values. Integer decoding rejects the whole event and session.
+ CPUUsage float64 `json:"cpu_usage"`
Memory int64 `json:"memory"`
Networks []trueNASAppNetworkStats `json:"networks"`
BlkIO trueNASAppBlkIOStats `json:"blkio"`
@@ -2740,7 +2742,7 @@ func (c *trueNASRPCClient) readAppStatsEvent(ctx context.Context, intervalSecond
}
appStats := AppStats{
- CPUPercent: float64(field.CPUUsage),
+ CPUPercent: field.CPUUsage,
MemoryBytes: field.Memory,
BlockReadBytes: field.BlkIO.Read,
BlockWriteBytes: field.BlkIO.Write,
diff --git a/internal/truenas/transport_test.go b/internal/truenas/transport_test.go
index 4dc03527f5..3b5c0b5b61 100644
--- a/internal/truenas/transport_test.go
+++ b/internal/truenas/transport_test.go
@@ -987,6 +987,118 @@ func TestJSONRPCFetchSnapshotUsesPublishedTrueNAS26MethodsWithoutREST(t *testing
}
}
+func TestJSONRPCAppStatsNumbersReuseSession(t *testing.T) {
+ for _, cpu := range []string{"0.0286", "0.0", "17"} {
+ t.Run(cpu, func(t *testing.T) {
+ var logins, subscribes, unsubscribes atomic.Int32
+ fixture := newProtocolFixture(t, func(_ int, request trueNASRPCRequest) protocolFixtureReply {
+ switch request.Method {
+ case "auth.login_ex":
+ logins.Add(1)
+ return protocolFixtureReply{result: map[string]any{"response_type": "SUCCESS"}}
+ case "app.query":
+ return protocolFixtureReply{result: []any{map[string]any{"id": "fixture-app", "name": "Fixture app", "state": "RUNNING"}}}
+ case "core.subscribe":
+ subscribes.Add(1)
+ collection := request.Params.([]any)[0].(string)
+ return protocolFixtureReply{result: "stats-sub", notifications: []protocolFixtureNotification{{
+ method: "collection_update", params: map[string]any{
+ "collection": collection, "fields": []any{map[string]any{
+ "app_name": "fixture-app", "cpu_usage": json.Number(cpu), "memory": 1072459776,
+ "networks": []any{map[string]any{"interface_name": "eth0", "rx_bytes": 3671, "tx_bytes": 284}},
+ "blkio": map[string]any{"read": 29720576, "write": 0},
+ }},
+ },
+ }}}
+ case "core.unsubscribe":
+ unsubscribes.Add(1)
+ return protocolFixtureReply{result: nil}
+ case "pool.query":
+ return protocolFixtureReply{result: []any{}}
+ default:
+ t.Errorf("unexpected method %s", request.Method)
+ return protocolFixtureReply{close: true}
+ }
+ }, nil)
+ client := protocolFixtureClient(t, fixture.server.URL, ClientConfig{APIKey: "fixture-key", Username: "fixture-user"})
+ wantCPU, _ := strconv.ParseFloat(cpu, 64)
+ var lastStats time.Time
+ for poll := 0; poll < 3; poll++ {
+ apps, err := client.GetApps(context.Background())
+ if err != nil || len(apps) != 1 || apps[0].ID != "fixture-app" {
+ t.Fatalf("poll %d lost app inventory: %+v, %v", poll, apps, err)
+ }
+ if stats := apps[0].Stats; stats == nil {
+ t.Errorf("poll %d lost app.stats for JSON CPU number %s", poll, cpu)
+ } else {
+ if stats.CPUPercent != wantCPU || stats.MemoryBytes != 1072459776 || stats.NetInRate != 3671 || stats.NetOutRate != 284 ||
+ stats.BlockReadBytes != 29720576 || stats.BlockWriteBytes != 0 || stats.IntervalSeconds != 2 || !stats.CollectedAt.After(lastStats) {
+ t.Errorf("poll %d lost precise CPU/sibling fields/freshness: %+v", poll, stats)
+ }
+ lastStats = stats.CollectedAt
+ }
+ }
+ if _, err := client.GetPools(context.Background()); err != nil {
+ t.Fatal(err)
+ }
+ if fixture.sessions.Load() != 1 || logins.Load() != 1 || subscribes.Load() != 3 || unsubscribes.Load() != 3 ||
+ fixture.restRequests.Load() != 0 || !client.TransportStatus().Connected {
+ t.Errorf("JSON CPU number caused session churn: sessions=%d logins=%d subscribe=%d unsubscribe=%d", fixture.sessions.Load(), logins.Load(), subscribes.Load(), unsubscribes.Load())
+ }
+ })
+ }
+}
+
+// Invalid CPU data must still fail the complete event, not fabricate an idle
+// sample or retry a non-transport decode failure after accepting JSON numbers.
+func TestJSONRPCAppStatsRejectsMalformedCPUNumbers(t *testing.T) {
+ for _, cpu := range []string{`"0.0286"`, `true`, `{}`, `[]`, `1e309`} {
+ t.Run(cpu, func(t *testing.T) {
+ var subscribes, unsubscribes, logins atomic.Int32
+ fixture := newProtocolFixture(t, func(_ int, request trueNASRPCRequest) protocolFixtureReply {
+ switch request.Method {
+ case "auth.login_ex":
+ logins.Add(1)
+ return protocolFixtureReply{result: map[string]any{"response_type": "SUCCESS"}}
+ case "core.subscribe":
+ subscribes.Add(1)
+ collection := request.Params.([]any)[0].(string)
+ return protocolFixtureReply{result: "stats-sub", notifications: []protocolFixtureNotification{{
+ method: "collection_update", params: map[string]any{
+ "collection": collection,
+ "fields": []any{
+ map[string]any{"app_name": "valid-app", "cpu_usage": json.Number("0.0286")},
+ map[string]any{"app_name": "invalid-app", "cpu_usage": json.RawMessage(cpu)},
+ },
+ },
+ }}}
+ case "core.unsubscribe":
+ unsubscribes.Add(1)
+ return protocolFixtureReply{result: nil}
+ case "pool.query":
+ return protocolFixtureReply{result: []any{}}
+ default:
+ t.Errorf("unexpected method %s", request.Method)
+ return protocolFixtureReply{close: true}
+ }
+ }, nil)
+ client := protocolFixtureClient(t, fixture.server.URL, ClientConfig{APIKey: "fixture-key", Username: "fixture-user"})
+ stats, err := client.GetAppStats(context.Background())
+ if err == nil || !strings.Contains(err.Error(), "decode truenas app.stats notification") || stats != nil {
+ t.Fatalf("malformed CPU produced partial/successful stats: %+v, %v", stats, err)
+ }
+ if client.rpc != nil || client.TransportStatus().Connected || fixture.sessions.Load() != 1 ||
+ logins.Load() != 1 || subscribes.Load() != 1 || unsubscribes.Load() != 0 {
+ t.Fatal("malformed stats were retried, retained the failed session or attempted unsubscribe")
+ }
+ if _, err := client.GetPools(context.Background()); err != nil || fixture.sessions.Load() != 2 ||
+ logins.Load() != 2 || fixture.restRequests.Load() != 0 {
+ t.Fatalf("next independent read did not recover over authenticated JSON-RPC: %v", err)
+ }
+ })
+ }
+}
+
func TestJSONRPCMalformedSubscriptionEventDiscardsSession(t *testing.T) {
fixture := newProtocolFixture(t, func(session int, request trueNASRPCRequest) protocolFixtureReply {
switch request.Method {