Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package truenas
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"net/url"
"strconv"
"strings"
"testing"
"time"
"github.com/gorilla/websocket"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
)
type apiResponse struct {
status int
body string
contentType string
}
type closeTrackingTransport struct {
closeCalls int
}
func (t *closeTrackingTransport) RoundTrip(*http.Request) (*http.Response, error) {
return nil, nil
}
func (t *closeTrackingTransport) CloseIdleConnections() {
t.closeCalls++
}
func TestClientGetters(t *testing.T) {
server := newMockServer(t, defaultAPIResponses(), nil)
t.Cleanup(server.Close)
client := mustClientForServer(t, server.URL, ClientConfig{APIKey: "api-key"})
ctx := context.Background()
system, err := client.GetSystemInfo(ctx)
if err != nil {
t.Fatalf("GetSystemInfo() error = %v", err)
}
if system.Hostname != "truenas-main" || system.Version != "TrueNAS-SCALE-24.10.2" {
t.Fatalf("unexpected system info: %+v", system)
}
if system.Build != "24.10.2.1" || system.UptimeSeconds != 86400 {
t.Fatalf("unexpected system fields: %+v", system)
}
if !system.Healthy || system.MachineID != "SER123" {
t.Fatalf("unexpected system health/identity: %+v", system)
}
if system.CPUCount != 16 || system.MemoryTotalBytes != 68719476736 {
t.Fatalf("unexpected system capacity mapping: %+v", system)
}
pools, err := client.GetPools(ctx)
if err != nil {
t.Fatalf("GetPools() error = %v", err)
}
if len(pools) != 1 {
t.Fatalf("expected 1 pool, got %d", len(pools))
}
if pools[0].ID != "1" || pools[0].Name != "tank" || pools[0].UsedBytes != 400 {
t.Fatalf("unexpected pool mapping: %+v", pools[0])
}
datasets, err := client.GetDatasets(ctx)
if err != nil {
t.Fatalf("GetDatasets() error = %v", err)
}
if len(datasets) != 1 {
t.Fatalf("expected 1 dataset, got %d", len(datasets))
}
if datasets[0].ID != "tank/apps" || datasets[0].UsedBytes != 12345 || datasets[0].AvailBytes != 555 {
t.Fatalf("unexpected dataset usage mapping: %+v", datasets[0])
}
if !datasets[0].Mounted || datasets[0].ReadOnly {
t.Fatalf("unexpected dataset mount/readonly mapping: %+v", datasets[0])
}
disks, err := client.GetDisks(ctx)
if err != nil {
t.Fatalf("GetDisks() error = %v", err)
}
if len(disks) != 2 {
t.Fatalf("expected 2 disks, got %d", len(disks))
}
if disks[0].Transport != "sata" || !disks[0].Rotational {
t.Fatalf("unexpected rotational disk mapping: %+v", disks[0])
}
if disks[1].Transport != "nvme" || disks[1].Rotational {
t.Fatalf("unexpected nvme disk mapping: %+v", disks[1])
}
if disks[0].Temperature != 34 || disks[1].Temperature != 49 {
t.Fatalf("unexpected disk temperatures: %+v", disks)
}
alerts, err := client.GetAlerts(ctx)
if err != nil {
t.Fatalf("GetAlerts() error = %v", err)
}
if len(alerts) != 1 {
t.Fatalf("expected 1 alert, got %d", len(alerts))
}
if alerts[0].ID != "a1" || alerts[0].Level != "WARNING" {
t.Fatalf("unexpected alert identity mapping: %+v", alerts[0])
}
if alerts[0].Datetime != time.UnixMilli(1707400000000).UTC() {
t.Fatalf("unexpected alert datetime: %s", alerts[0].Datetime)
}
apps, err := client.GetApps(ctx)
if err != nil {
t.Fatalf("GetApps() error = %v", err)
}
if len(apps) != 1 {
t.Fatalf("expected 1 app, got %d", len(apps))
}
if apps[0].ID != "nextcloud" || apps[0].Name != "Nextcloud" {
t.Fatalf("unexpected app identity mapping: %+v", apps[0])
}
if apps[0].ContainerCount != 2 || len(apps[0].Containers) != 2 {
t.Fatalf("unexpected app container mapping: %+v", apps[0])
}
if len(apps[0].UsedPorts) != 1 || apps[0].UsedPorts[0].ContainerPort != 443 {
t.Fatalf("unexpected app used ports mapping: %+v", apps[0].UsedPorts)
}
if len(apps[0].Volumes) != 2 || len(apps[0].Networks) != 1 {
t.Fatalf("unexpected app volume/network mapping: volumes=%d networks=%d", len(apps[0].Volumes), len(apps[0].Networks))
}
shares, err := client.GetNetworkShares(ctx)
if err != nil {
t.Fatalf("GetNetworkShares() error = %v", err)
}
if len(shares) != 2 {
t.Fatalf("expected 2 network shares, got %d", len(shares))
}
if shares[0].Name != "Media" || shares[0].Protocol != "SMB" || shares[0].Dataset != "tank/media" || !shares[0].AuditEnabled {
t.Fatalf("unexpected SMB share mapping: %+v", shares[0])
}
if shares[1].Name != "tank/projects" || shares[1].Protocol != "NFS" || !shares[1].ReadOnly || len(shares[1].Networks) != 1 {
t.Fatalf("unexpected NFS share mapping: %+v", shares[1])
}
}
func TestClientAuthHeaderAPIKey(t *testing.T) {
server := newMockServer(t, map[string]apiResponse{
"/api/v2.0/system/info": {body: `{"hostname":"nas","version":"TrueNAS-SCALE-24.10.2","buildtime":"b","uptime_seconds":1}`},
}, func(t *testing.T, request *http.Request) {
if request.URL.Path == "/api/current" {
return
}
if got := request.Header.Get("Authorization"); got != "Bearer test-key" {
t.Fatalf("expected bearer auth header, got %q", got)
}
})
t.Cleanup(server.Close)
client := mustClientForServer(t, server.URL, ClientConfig{APIKey: "test-key"})
if err := client.TestConnection(context.Background()); err != nil {
t.Fatalf("TestConnection() error = %v", err)
}
}
func TestClientAuthHeaderBasic(t *testing.T) {
server := newMockServer(t, map[string]apiResponse{
"/api/v2.0/system/info": {body: `{"hostname":"nas","version":"TrueNAS-SCALE-24.10.2","buildtime":"b","uptime_seconds":1}`},
}, func(t *testing.T, request *http.Request) {
if request.URL.Path == "/api/current" {
return
}
username, password, ok := request.BasicAuth()
if !ok {
t.Fatalf("expected basic auth")
}
if username != "admin" || password != "secret" {
t.Fatalf("unexpected basic auth credentials: %q:%q", username, password)
}
})
t.Cleanup(server.Close)
client := mustClientForServer(t, server.URL, ClientConfig{Username: "admin", Password: "secret"})
if err := client.TestConnection(context.Background()); err != nil {
t.Fatalf("TestConnection() error = %v", err)
}
}
func TestGetSystemInfoAcceptsStructuredBuildTime(t *testing.T) {
var response systemInfoResponse
if err := json.Unmarshal([]byte(`{"hostname":"nas","version":"TrueNAS-SCALE-25.10.3.1","buildtime":{"$date":"2026-05-14T18:24:01+02:00"},"uptime_seconds":1}`), &response); err != nil {
t.Fatalf("Unmarshal() error = %v", err)
}
system := systemInfoFromResponse(response)
if system.Build != "2026-05-14T18:24:01+02:00" {
t.Fatalf("Build = %q, want structured buildtime date", system.Build)
}
}
func TestGetSystemInfoAcceptsFractionalUptimeSeconds(t *testing.T) {
var response systemInfoResponse
if err := json.Unmarshal([]byte(`{"hostname":"nas","version":"TrueNAS-SCALE-25.10.3.1","buildtime":{"$date":"2026-05-14T18:24:01+02:00"},"uptime_seconds":360144.629139547}`), &response); err != nil {
t.Fatalf("Unmarshal() error = %v", err)
}
system := systemInfoFromResponse(response)
if system.UptimeSeconds != 360144 {
t.Fatalf("UptimeSeconds = %d, want truncated fractional uptime", system.UptimeSeconds)
}
}
func TestTestConnectionSuccessAndFailure(t *testing.T) {
successServer := newMockServer(t, map[string]apiResponse{
"/api/v2.0/system/info": {body: `{"hostname":"nas","version":"TrueNAS-SCALE-24.10.2","buildtime":"b","uptime_seconds":1}`},
}, nil)
t.Cleanup(successServer.Close)
successClient := mustClientForServer(t, successServer.URL, ClientConfig{APIKey: "key"})
if err := successClient.TestConnection(context.Background()); err != nil {
t.Fatalf("TestConnection() success error = %v", err)
}
failureServer := newMockServer(t, map[string]apiResponse{
"/api/v2.0/system/info": {status: http.StatusUnauthorized, body: `{"error":"unauthorized"}`},
}, nil)
t.Cleanup(failureServer.Close)
failureClient := mustClientForServer(t, failureServer.URL, ClientConfig{APIKey: "bad"})
if err := failureClient.TestConnection(context.Background()); err == nil {
t.Fatal("expected TestConnection() failure error")
}
}
func TestFetchSnapshot(t *testing.T) {
server := newMockServer(t, defaultAPIResponses(), nil)
t.Cleanup(server.Close)
client := mustClientForServer(t, server.URL, ClientConfig{APIKey: "api-key"})
snapshot, err := client.FetchSnapshot(context.Background())
if err != nil {
t.Fatalf("FetchSnapshot() error = %v", err)
}
if snapshot == nil {
t.Fatal("expected snapshot")
}
if snapshot.CollectedAt.IsZero() {
t.Fatal("expected non-zero CollectedAt")
}
if snapshot.System.Hostname != "truenas-main" {
t.Fatalf("unexpected snapshot system: %+v", snapshot.System)
}
if len(snapshot.Pools) != 1 || len(snapshot.Datasets) != 1 || len(snapshot.Disks) != 2 || len(snapshot.Alerts) != 1 || len(snapshot.Services) != 2 || len(snapshot.Apps) != 1 || len(snapshot.VMs) != 1 || len(snapshot.Shares) != 2 {
t.Fatalf("unexpected snapshot counts: pools=%d datasets=%d disks=%d alerts=%d services=%d apps=%d vms=%d shares=%d",
len(snapshot.Pools), len(snapshot.Datasets), len(snapshot.Disks), len(snapshot.Alerts), len(snapshot.Services), len(snapshot.Apps), len(snapshot.VMs), len(snapshot.Shares))
}
if snapshot.Disks[0].Temperature != 34 || snapshot.Disks[1].Temperature != 49 {
t.Fatalf("unexpected snapshot disk temperatures: %+v", snapshot.Disks)
}
if snapshot.Apps[0].ID != "nextcloud" || snapshot.Apps[0].ContainerCount != 2 {
t.Fatalf("unexpected snapshot apps: %+v", snapshot.Apps)
}
if snapshot.VMs[0].Name != "windows-lab" || snapshot.VMs[0].MemoryBytes != 8*1024*1024*1024 {
t.Fatalf("unexpected snapshot vms: %+v", snapshot.VMs)
}
if snapshot.Services[0].Service != "smb" || !snapshot.Services[0].Enabled || snapshot.Services[0].State != "RUNNING" {
t.Fatalf("unexpected snapshot services: %+v", snapshot.Services)
}
}
func TestGetPoolsDatasetsAndAlertsUseNativeQueryShapes(t *testing.T) {
server := newMockServerWithRPC(t, map[string]apiResponse{
"/api/v2.0/pool": {status: http.StatusInternalServerError, body: `{"error":"legacy pool endpoint should not be used"}`},
"/api/v2.0/pool/dataset": {status: http.StatusInternalServerError, body: `{"error":"legacy dataset endpoint should not be used"}`},
"/api/v2.0/alert/list": {status: http.StatusInternalServerError, body: `{"error":"legacy alert endpoint should not be used"}`},
}, nil, func(t *testing.T, conn *websocket.Conn) {
authReq := readRPCRequest(t, conn)
if authReq.Method != "auth.login_with_api_key" {
t.Fatalf("expected api-key auth method, got %q", authReq.Method)
}
writeRPCResult(t, conn, authReq.ID, true)
request := readRPCRequest(t, conn)
switch request.Method {
case "pool.query":
assertQueryParams(t, request.Params, "pool.query")
writeRPCResult(t, conn, request.ID, defaultRoutePayloadMaps(t, "/api/v2.0/pool"))
case "pool.dataset.query":
assertQueryParams(t, request.Params, "pool.dataset.query")
writeRPCResult(t, conn, request.ID, defaultRoutePayloadMaps(t, "/api/v2.0/pool/dataset"))
case "alert.list":
params, ok := request.Params.([]any)
if !ok || len(params) != 0 {
t.Fatalf("expected alert.list to use no params, got %#v", request.Params)
}
writeRPCResult(t, conn, request.ID, []map[string]any{{
"id": "a1", "level": "WARNING", "formatted": "53 uncorrectable errors reported", "source": "SMART",
"klass": "SMARTUncorrectedErrorsAlert", "args": map[string]any{"ue": 53, "name": "/dev/sda", "serial": "SER-A"},
"dismissed": false, "datetime": map[string]any{"$date": 1707400000000},
}})
default:
t.Fatalf("unexpected rpc method %q", request.Method)
}
})
t.Cleanup(server.Close)
client := mustClientForServer(t, server.URL, ClientConfig{APIKey: "api-key"})
pools, err := client.GetPools(context.Background())
if err != nil {
t.Fatalf("GetPools() error = %v", err)
}
if len(pools) != 1 || pools[0].Name != "tank" || pools[0].UsedBytes != 400 {
t.Fatalf("unexpected native pool mapping: %+v", pools)
}
datasets, err := client.GetDatasets(context.Background())
if err != nil {
t.Fatalf("GetDatasets() error = %v", err)
}
if len(datasets) != 1 || datasets[0].ID != "tank/apps" || datasets[0].Pool != "tank" || datasets[0].UsedBytes != 12345 {
t.Fatalf("unexpected native dataset mapping: %+v", datasets)
}
alerts, err := client.GetAlerts(context.Background())
if err != nil {
t.Fatalf("GetAlerts() error = %v", err)
}
if len(alerts) != 1 || alerts[0].ID != "a1" || alerts[0].Level != "WARNING" ||
alerts[0].Class != "SMARTUncorrectedErrorsAlert" || alerts[0].DiskName != "/dev/sda" || alerts[0].DiskSerial != "SER-A" ||
!alerts[0].SMARTUncorrectedReported || alerts[0].SMARTUncorrectedErrors != 53 {
t.Fatalf("unexpected native alert mapping: %+v", alerts)
}
}
func TestGetAlertsParsesLegacyDiskIdentityArguments(t *testing.T) {
responses := defaultAPIResponses()
responses["/api/v2.0/alert/list"] = apiResponse{
body: `[{"id":"smart-1","level":"WARNING","formatted":"53 uncorrectable errors reported for /dev/sda (SER-A).","source":"SMART","klass":"SMARTUncorrectedErrorsAlert","args":{"ue":53,"name":"/dev/sda","serial":"SER-A"},"dismissed":true,"datetime":{"$date":1707400000000}}]`,
}
server := newMockServer(t, responses, nil)
t.Cleanup(server.Close)
client := mustClientForServer(t, server.URL, ClientConfig{APIKey: "api-key"})
alerts, err := client.GetAlerts(context.Background())
if err != nil {
t.Fatalf("GetAlerts() error = %v", err)
}
if len(alerts) != 1 || alerts[0].Class != "SMARTUncorrectedErrorsAlert" ||
alerts[0].DiskName != "/dev/sda" || alerts[0].DiskSerial != "SER-A" ||
!alerts[0].SMARTUncorrectedReported || alerts[0].SMARTUncorrectedErrors != 53 || !alerts[0].Dismissed {
t.Fatalf("unexpected legacy SMART alert mapping: %+v", alerts)
}
}
func TestSMARTAlertEvidenceFromArgsUsesOnlyTypedNativeFields(t *testing.T) {
tests := []struct {
name string
class string
args map[string]any
wantUncorrected int64
wantUncorrectedSeen bool
wantSpare int
wantSpareSeen bool
}{
{
name: "uncorrected counter", class: "SMARTUncorrectedErrors",
args: map[string]any{"ue": float64(53)}, wantUncorrected: 53, wantUncorrectedSeen: true,
},
{
name: "spare reserve", class: "SMARTSpareBlockCountAlert",
args: map[string]any{"sb": "8"}, wantSpare: 8, wantSpareSeen: true,
},
{
name: "zero spare reserve", class: "SMARTSpareBlockCount",
args: map[string]any{"sb": json.Number("0")}, wantSpareSeen: true,
},
{
name: "maximum spare reserve", class: "SMARTSpareBlockCountAlert",
args: map[string]any{"sb": map[string]any{"parsed": int64(100)}}, wantSpare: 100, wantSpareSeen: true,
},
{name: "negative counter rejected", class: "SMARTUncorrectedErrors", args: map[string]any{"ue": -1}},
{name: "out of range percentage rejected", class: "SMARTSpareBlockCount", args: map[string]any{"sb": 101}},
{name: "oversized percentage rejected", class: "SMARTSpareBlockCount", args: map[string]any{"sb": "9223372036854775807"}},
{name: "fractional percentage rejected", class: "SMARTSpareBlockCount", args: map[string]any{"sb": 8.5}},
{name: "formatted text is not parsed", class: "SMARTUncorrectedErrors", args: map[string]any{"text": "53 errors"}},
{name: "unrelated alert cannot inject smart data", class: "DiskTemperatureAlert", args: map[string]any{"ue": 53, "sb": 8}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
uncorrected, uncorrectedSeen, spare, spareSeen := smartAlertEvidenceFromArgs(tt.class, tt.args)
if uncorrected != tt.wantUncorrected || uncorrectedSeen != tt.wantUncorrectedSeen ||
spare != tt.wantSpare || spareSeen != tt.wantSpareSeen {
t.Fatalf("smartAlertEvidenceFromArgs() = (%d,%v,%d,%v), want (%d,%v,%d,%v)",
uncorrected, uncorrectedSeen, spare, spareSeen,
tt.wantUncorrected, tt.wantUncorrectedSeen, tt.wantSpare, tt.wantSpareSeen)
}
})
}
}
func TestGetServicesUsesNativeServiceQueryShape(t *testing.T) {
server := newMockServerWithRPC(t, map[string]apiResponse{
"/api/v2.0/service": {status: http.StatusInternalServerError, body: `{"error":"legacy service endpoint should not be used"}`},
}, nil, func(t *testing.T, conn *websocket.Conn) {
authReq := readRPCRequest(t, conn)
if authReq.Method != "auth.login_with_api_key" {
t.Fatalf("expected api-key auth method, got %q", authReq.Method)
}
writeRPCResult(t, conn, authReq.ID, true)
queryReq := readRPCRequest(t, conn)
if queryReq.Method != "service.query" {
t.Fatalf("expected service.query, got %q", queryReq.Method)
}
params, ok := queryReq.Params.([]any)
if !ok || len(params) != 2 {
t.Fatalf("expected service.query filters/options params, got %#v", queryReq.Params)
}
options, ok := params[1].(map[string]any)
if !ok {
t.Fatalf("expected service.query options, got %#v", params[1])
}
extra, ok := options["extra"].(map[string]any)
if !ok || extra["include_state"] != true {
t.Fatalf("expected service.query include_state option, got %#v", options)
}
writeRPCResult(t, conn, queryReq.ID, []map[string]any{
{"id": 1, "service": "smb", "enable": true, "state": "RUNNING", "pids": []int{2418, 2420}},
{"id": 2, "service": "ssh", "enable": false, "state": "STOPPED", "pids": []any{}},
})
})
t.Cleanup(server.Close)
client := mustClientForServer(t, server.URL, ClientConfig{APIKey: "api-key"})
services, err := client.GetServices(context.Background())
if err != nil {
t.Fatalf("GetServices() error = %v", err)
}
if len(services) != 2 {
t.Fatalf("expected 2 services, got %d", len(services))
}
if services[0].ID != "1" || services[0].Service != "smb" || !services[0].Enabled || services[0].State != "RUNNING" || len(services[0].PIDs) != 2 {
t.Fatalf("unexpected smb service mapping: %+v", services[0])
}
if services[1].Service != "ssh" || services[1].Enabled || services[1].State != "STOPPED" {
t.Fatalf("unexpected ssh service mapping: %+v", services[1])
}
}
func TestGetDisksUsesNativeDiskQueryShape(t *testing.T) {
connectionCount := 0
server := newMockServerWithRPC(t, map[string]apiResponse{
"/api/v2.0/disk": {status: http.StatusInternalServerError, body: `{"error":"legacy disk endpoint should not be used"}`},
"/api/v2.0/disk/temperatures": {
body: `{"sda":34}`,
},
}, nil, func(t *testing.T, conn *websocket.Conn) {
authReq := readRPCRequest(t, conn)
if authReq.Method != "auth.login_with_api_key" {
t.Fatalf("expected api-key auth method, got %q", authReq.Method)
}
writeRPCResult(t, conn, authReq.ID, true)
connectionCount++
request := readRPCRequest(t, conn)
switch connectionCount {
case 1:
if request.Method != "disk.query" {
t.Fatalf("expected disk.query, got %q", request.Method)
}
params, ok := request.Params.([]any)
if !ok || len(params) != 2 {
t.Fatalf("expected disk.query filters/options params, got %#v", request.Params)
}
payload := defaultRoutePayloadMaps(t, "/api/v2.0/disk")
payload[0]["smart_status"] = "FAILED"
writeRPCResult(t, conn, request.ID, payload[:1])
case 2:
if request.Method != "reporting.get_data" {
t.Fatalf("expected reporting.get_data, got %q", request.Method)
}
writeRPCResult(t, conn, request.ID, []map[string]any{{
"name": "disktemp",
"identifier": "sda",
"legend": []string{"time", "temperature"},
"aggregations": map[string]any{
"mean": map[string]any{"temperature": 34.0},
},
}})
case 3:
if request.Method != "disk.temperature_agg" {
t.Fatalf("expected disk.temperature_agg, got %q", request.Method)
}
writeRPCResult(t, conn, request.ID, map[string]any{
"sda": map[string]any{
"min": 29.0,
"avg": 32.8,
"max": 38.0,
"window_days": 7,
},
})
default:
t.Fatalf("unexpected extra websocket connection %d", connectionCount)
}
})
t.Cleanup(server.Close)
client := mustClientForServer(t, server.URL, ClientConfig{APIKey: "api-key"})
disks, err := client.GetDisks(context.Background())
if err != nil {
t.Fatalf("GetDisks() error = %v", err)
}
if len(disks) != 1 {
t.Fatalf("expected 1 disk, got %d", len(disks))
}
if disks[0].Name != "sda" || disks[0].Temperature != 34 || disks[0].TemperatureAggregate.MaxCelsius != 38 || disks[0].Health != "FAILED" || !disks[0].HealthStatusPresent {
t.Fatalf("unexpected native disk mapping: %+v", disks[0])
}
}
func TestGetDisksParsesRESTSmartStatus(t *testing.T) {
server := newMockServer(t, map[string]apiResponse{
"/api/v2.0/disk": {
body: `[
{"identifier":"{disk-1}","name":"sda","serial":"SER-A","size":1000000,"model":"Seagate","type":"HDD","pool":"tank","bus":"SATA","rotationrate":7200,"status":"ONLINE","smart_status":null},
{"identifier":"{disk-2}","name":"sdb","serial":"SER-B","size":1000000,"model":"Seagate","type":"HDD","pool":"tank","bus":"SATA","rotationrate":7200,"status":"ONLINE","smart_status":""},
{"identifier":"{disk-3}","name":"sdc","serial":"SER-C","size":1000000,"model":"Seagate","type":"HDD","pool":"tank","bus":"SATA","rotationrate":7200,"status":"ONLINE","smart_status":"FAILED"},
{"identifier":"{disk-4}","name":"sdd","serial":"SER-D","size":1000000,"model":"Seagate","type":"HDD","pool":"tank","bus":"SATA","rotationrate":7200,"status":"ONLINE"}
]`,
},
"/api/v2.0/disk/temperatures": {
body: `{}`,
},
}, nil)
t.Cleanup(server.Close)
client := mustClientForServer(t, server.URL, ClientConfig{APIKey: "api-key"})
disks, err := client.GetDisks(context.Background())
if err != nil {
t.Fatalf("GetDisks() error = %v", err)
}
byName := make(map[string]Disk, len(disks))
for _, disk := range disks {
byName[disk.Name] = disk
}
tests := []struct {
name string
wantHealth string
wantPresent bool
}{
{name: "sda", wantHealth: "UNKNOWN", wantPresent: true},
{name: "sdb", wantHealth: "UNKNOWN", wantPresent: true},
{name: "sdc", wantHealth: "FAILED", wantPresent: true},
{name: "sdd", wantHealth: "", wantPresent: false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
disk, ok := byName[tt.name]
if !ok {
t.Fatalf("missing disk %q in %+v", tt.name, disks)
}
if disk.Health != tt.wantHealth || disk.HealthStatusPresent != tt.wantPresent {
t.Fatalf("disk %s health=(%q,%v), want (%q,%v)", tt.name, disk.Health, disk.HealthStatusPresent, tt.wantHealth, tt.wantPresent)
}
})
}
}
func TestGetVMsParsesNativeVMQueryShape(t *testing.T) {
server := newMockServerWithRPC(t, map[string]apiResponse{}, nil, func(t *testing.T, conn *websocket.Conn) {
authReq := readRPCRequest(t, conn)
if authReq.Method != "auth.login_with_api_key" {
t.Fatalf("expected api-key auth method, got %q", authReq.Method)
}
writeRPCResult(t, conn, authReq.ID, true)
queryReq := readRPCRequest(t, conn)
if queryReq.Method != "vm.query" {
t.Fatalf("expected vm.query, got %q", queryReq.Method)
}
writeRPCResult(t, conn, queryReq.ID, []map[string]any{
{
"id": 42,
"name": "windows-lab",
"description": "Build test box",
"vcpus": 4,
"cores": 2,
"threads": 2,
"memory": 8192,
"min_memory": 4096,
"cpu_mode": "HOST-PASSTHROUGH",
"cpu_model": nil,
"bootloader": "UEFI",
"autostart": true,
"suspend_on_snapshot": true,
"trusted_platform_module": true,
"enable_secure_boot": true,
"time": "UTC",
"arch_type": "x86_64",
"machine_type": "q35",
"uuid": "vm-uuid-1",
"display_available": true,
"status": map[string]any{
"state": "RUNNING",
"pid": 1234,
"domain_state": "RUNNING",
},
"devices": []map[string]any{
{"id": 1, "attributes": map[string]any{"dtype": "DISK"}},
{"id": 2, "attributes": map[string]any{"dtype": "NIC"}},
{"id": 3, "attributes": map[string]any{"dtype": "DISPLAY"}},
{"id": 4, "attributes": map[string]any{"dtype": "CDROM"}},
},
},
})
})
t.Cleanup(server.Close)
client := mustClientForServer(t, server.URL, ClientConfig{APIKey: "api-key"})
vms, err := client.GetVMs(context.Background())
if err != nil {
t.Fatalf("GetVMs() error = %v", err)
}
if len(vms) != 1 {
t.Fatalf("expected 1 VM, got %d", len(vms))
}
vm := vms[0]
if vm.ID != "42" || vm.Name != "windows-lab" || vm.State != "RUNNING" || vm.DomainState != "RUNNING" {
t.Fatalf("unexpected VM identity/status: %+v", vm)
}
if vm.VCPUs != 4 || vm.Cores != 2 || vm.Threads != 2 {
t.Fatalf("unexpected VM CPU topology: %+v", vm)
}
if vm.MemoryBytes != 8*1024*1024*1024 || vm.MinMemoryBytes != 4*1024*1024*1024 {
t.Fatalf("unexpected VM memory: %+v", vm)
}
if !vm.Autostart || !vm.SuspendOnSnapshot || !vm.TrustedPlatformModule || !vm.SecureBoot {
t.Fatalf("expected VM boolean flags, got %+v", vm)
}
if vm.DeviceCount != 4 || vm.DiskCount != 1 || vm.NICCount != 1 || vm.DisplayCount != 1 || vm.CDROMCount != 1 {
t.Fatalf("unexpected VM device counts: %+v", vm)
}
}
func TestGetAppsParsesNativeAppQueryShape(t *testing.T) {
server := newMockServerWithRPC(t, map[string]apiResponse{
"/api/v2.0/app": {status: http.StatusInternalServerError, body: `{"error":"legacy app endpoint should not be used"}`},
}, nil, func(t *testing.T, conn *websocket.Conn) {
authReq := readRPCRequest(t, conn)
if authReq.Method != "auth.login_with_api_key" {
t.Fatalf("expected api-key auth method, got %q", authReq.Method)
}
writeRPCResult(t, conn, authReq.ID, true)
request := readRPCRequest(t, conn)
switch request.Method {
case "app.query":
params, ok := request.Params.([]any)
if !ok || len(params) != 2 {
t.Fatalf("expected app.query filters/options params, got %#v", request.Params)
}
writeRPCResult(t, conn, request.ID, defaultAppQueryPayload(t))
case "core.subscribe":
writeRPCError(t, conn, request.ID, 1, "stats unavailable")
default:
t.Fatalf("unexpected rpc method %q", request.Method)
}
})
t.Cleanup(server.Close)
client := mustClientForServer(t, server.URL, ClientConfig{APIKey: "api-key"})
apps, err := client.GetApps(context.Background())
if err != nil {
t.Fatalf("GetApps() error = %v", err)
}
if len(apps) != 1 {
t.Fatalf("expected 1 app, got %d", len(apps))
}
app := apps[0]
if app.ID != "nextcloud" || app.Name != "Nextcloud" || app.State != "RUNNING" {
t.Fatalf("unexpected app identity/status: %+v", app)
}
if app.ContainerCount != 2 || len(app.Containers) != 2 {
t.Fatalf("unexpected app container mapping: %+v", app)
}
if len(app.UsedPorts) != 1 || len(app.UsedPorts[0].HostPorts) != 1 || app.UsedPorts[0].HostPorts[0].HostPort != 30443 {
t.Fatalf("unexpected app port mapping: %+v", app.UsedPorts)
}
if len(app.Volumes) != 2 || len(app.Images) != 2 || len(app.Networks) != 1 {
t.Fatalf("unexpected app runtime shape: volumes=%d images=%d networks=%d", len(app.Volumes), len(app.Images), len(app.Networks))
}
}
func TestGetAppsEnrichesStatsFromRPC(t *testing.T) {
server := newMockServerWithRPC(t, defaultAPIResponses(), nil, func(t *testing.T, conn *websocket.Conn) {
authReq := readRPCRequest(t, conn)
if authReq.Method != "auth.login_with_api_key" {
t.Fatalf("expected api-key auth method, got %q", authReq.Method)
}
writeRPCResult(t, conn, authReq.ID, true)
request := readRPCRequest(t, conn)
switch request.Method {
case "app.query":
writeRPCResult(t, conn, request.ID, defaultAppQueryPayload(t))
return
case "core.subscribe":
default:
t.Fatalf("expected app.query or core.subscribe, got %q", request.Method)
}
writeRPCResult(t, conn, request.ID, "sub-1")
writeRPCNotification(t, conn, "collection_update", map[string]any{
"collection": "app.stats:{\"interval\":2}",
"fields": []map[string]any{
{
"app_name": "nextcloud",
"cpu_usage": 17,
"memory": 268435456,
"networks": []map[string]any{
{"interface_name": "eth0", "rx_bytes": 2048, "tx_bytes": 1024},
{"interface_name": "eth1", "rx_bytes": 512, "tx_bytes": 256},
},
"blkio": map[string]any{
"read": 4096,
"write": 2048,
},
},
},
})
expectRPCUnsubscribe(t, conn, "sub-1")
})
t.Cleanup(server.Close)
client := mustClientForServer(t, server.URL, ClientConfig{APIKey: "api-key"})
apps, err := client.GetApps(context.Background())
if err != nil {
t.Fatalf("GetApps() error = %v", err)
}
if len(apps) != 1 || apps[0].Stats == nil {
t.Fatalf("expected one app with stats, got %+v", apps)
}
if apps[0].Stats.CPUPercent != 17 || apps[0].Stats.MemoryBytes != 268435456 {
t.Fatalf("unexpected app stats core fields: %+v", apps[0].Stats)
}
if apps[0].Stats.NetInRate != 2560 || apps[0].Stats.NetOutRate != 1280 {
t.Fatalf("unexpected aggregated app network rates: %+v", apps[0].Stats)
}
if apps[0].Stats.BlockReadBytes != 4096 || apps[0].Stats.BlockWriteBytes != 2048 {
t.Fatalf("unexpected app blkio stats: %+v", apps[0].Stats)
}
if len(apps[0].Stats.Interfaces) != 2 {
t.Fatalf("expected two interface stats, got %+v", apps[0].Stats.Interfaces)
}
}
func TestGetZFSSnapshotsUsesNativeSnapshotQueryShape(t *testing.T) {
server := newMockServerWithRPC(t, map[string]apiResponse{
"/api/v2.0/zfs/snapshot": {status: http.StatusInternalServerError, body: `{"error":"legacy snapshot endpoint should not be used"}`},
}, nil, func(t *testing.T, conn *websocket.Conn) {
authReq := readRPCRequest(t, conn)
if authReq.Method != "auth.login_with_api_key" {
t.Fatalf("expected api-key auth method, got %q", authReq.Method)
}
writeRPCResult(t, conn, authReq.ID, true)
request := readRPCRequest(t, conn)
if request.Method != "zfs.resource.snapshot.query" {
t.Fatalf("expected zfs.resource.snapshot.query, got %q", request.Method)
}
params, ok := request.Params.([]any)
if !ok || len(params) != 1 {
t.Fatalf("unexpected snapshot query params: %#v", request.Params)
}
writeRPCResult(t, conn, request.ID, []map[string]any{{
"name": "tank/apps@auto-20260331-0600",
"dataset": "tank/apps",
"snapshot_name": "auto-20260331-0600",
"properties": map[string]any{
"creation": map[string]any{"raw": "1710000000"},
"used": map[string]any{"raw": "1024"},
"referenced": map[string]any{"raw": "2048"},
},
}})
})
t.Cleanup(server.Close)
client := mustClientForServer(t, server.URL, ClientConfig{APIKey: "api-key"})
snapshots, err := client.GetZFSSnapshots(context.Background())
if err != nil {
t.Fatalf("GetZFSSnapshots() error = %v", err)
}
if len(snapshots) != 1 {
t.Fatalf("expected 1 snapshot, got %d", len(snapshots))
}
snapshot := snapshots[0]
if snapshot.FullName != "tank/apps@auto-20260331-0600" || snapshot.Dataset != "tank/apps" || snapshot.Name != "auto-20260331-0600" {
t.Fatalf("unexpected snapshot identity: %+v", snapshot)
}
if snapshot.CreatedAt == nil || !snapshot.CreatedAt.Equal(time.Unix(1710000000, 0).UTC()) {
t.Fatalf("unexpected snapshot creation time: %+v", snapshot.CreatedAt)
}
if snapshot.UsedBytes == nil || *snapshot.UsedBytes != 1024 || snapshot.Referenced == nil || *snapshot.Referenced != 2048 {
t.Fatalf("unexpected snapshot byte properties: %+v", snapshot)
}
}
func TestGetReplicationTasksUsesNativeReplicationQueryShape(t *testing.T) {
server := newMockServerWithRPC(t, map[string]apiResponse{
"/api/v2.0/replication": {status: http.StatusInternalServerError, body: `{"error":"legacy replication endpoint should not be used"}`},
}, nil, func(t *testing.T, conn *websocket.Conn) {
authReq := readRPCRequest(t, conn)
if authReq.Method != "auth.login_with_api_key" {
t.Fatalf("expected api-key auth method, got %q", authReq.Method)
}
writeRPCResult(t, conn, authReq.ID, true)
request := readRPCRequest(t, conn)
if request.Method != "replication.query" {
t.Fatalf("expected replication.query, got %q", request.Method)
}
params, ok := request.Params.([]any)
if !ok || len(params) != 2 {
t.Fatalf("expected replication.query filters/options params, got %#v", request.Params)
}
writeRPCResult(t, conn, request.ID, []map[string]any{{
"id": 7,
"name": "Offsite apps",
"direction": "PUSH",
"transport": "SSH",
"readonly": "SET",
"source_datasets": []string{"tank/apps"},
"target_dataset": "backup/apps",
"ssh_credentials": map[string]any{
"id": 3,
"name": "backup-nas",
"attributes": map[string]any{
"host": "backup.example.test",
"port": 22,
},
},
"state": map[string]any{
"state": "SUCCESS",
"datetime": "2026-03-31T06:30:00Z",
"last_snapshot": "tank/apps@auto-20260331-0600",
},
}})
})
t.Cleanup(server.Close)
client := mustClientForServer(t, server.URL, ClientConfig{APIKey: "api-key"})
tasks, err := client.GetReplicationTasks(context.Background())
if err != nil {
t.Fatalf("GetReplicationTasks() error = %v", err)
}
if len(tasks) != 1 {
t.Fatalf("expected 1 task, got %d", len(tasks))
}
task := tasks[0]
if task.ID != "7" || task.Name != "Offsite apps" || task.Direction != "PUSH" {
t.Fatalf("unexpected replication identity: %+v", task)
}
if len(task.SourceDatasets) != 1 || task.SourceDatasets[0] != "tank/apps" || task.TargetDataset != "backup/apps" {
t.Fatalf("unexpected replication datasets: %+v", task)
}
if task.Transport != "SSH" || task.ReadOnlyMode != "SET" || task.TargetHost != "backup.example.test" {
t.Fatalf("unexpected replication target posture: %+v", task)
}
if task.LastState != "SUCCESS" || task.LastSnapshot != "tank/apps@auto-20260331-0600" {
t.Fatalf("unexpected replication state: %+v", task)
}
if task.LastRun == nil || task.LastRun.Format(time.RFC3339) != "2026-03-31T06:30:00Z" {
t.Fatalf("unexpected replication last run: %+v", task.LastRun)
}
}
func TestStartAndStopAppUseRPCMethods(t *testing.T) {
var rpcSessions int
server := newMockServerWithRPC(t, defaultAPIResponses(), nil, func(t *testing.T, conn *websocket.Conn) {
rpcSessions++
authReq := readRPCRequest(t, conn)
if authReq.Method != "auth.login_with_api_key" {
t.Fatalf("expected api-key auth method, got %q", authReq.Method)
}
writeRPCResult(t, conn, authReq.ID, true)
for _, expectedMethod := range []string{"app.start", "app.stop"} {
actionReq := readRPCRequest(t, conn)
if actionReq.Method != expectedMethod {
t.Fatalf("expected %s, got %q", expectedMethod, actionReq.Method)
}
writeRPCResult(t, conn, actionReq.ID, true)
}
})
t.Cleanup(server.Close)
client := mustClientForServer(t, server.URL, ClientConfig{APIKey: "api-key"})
if err := client.StartApp(context.Background(), "nextcloud"); err != nil {
t.Fatalf("StartApp() error = %v", err)
}
if err := client.StopApp(context.Background(), "nextcloud"); err != nil {
t.Fatalf("StopApp() error = %v", err)
}
if rpcSessions != 1 {
t.Fatalf("expected one persistent RPC app-action session, got %d", rpcSessions)
}
}
func TestGetAppLogsUsesRPCSubscription(t *testing.T) {
server := newMockServerWithRPC(t, defaultAPIResponses(), nil, func(t *testing.T, conn *websocket.Conn) {
authReq := readRPCRequest(t, conn)
if authReq.Method != "auth.login_with_api_key" {
t.Fatalf("expected api-key auth method, got %q", authReq.Method)
}
writeRPCResult(t, conn, authReq.ID, true)
subscribeReq := readRPCRequest(t, conn)
if subscribeReq.Method != "core.subscribe" {
t.Fatalf("expected core.subscribe, got %q", subscribeReq.Method)
}
params, ok := subscribeReq.Params.([]any)
if !ok || len(params) != 1 {
t.Fatalf("expected one subscription param, got %#v", subscribeReq.Params)
}
subscriptionName, _ := params[0].(string)
if !strings.HasPrefix(subscriptionName, "app.container_log_follow:") {
t.Fatalf("expected app.container_log_follow subscription, got %q", subscriptionName)
}
if !strings.Contains(subscriptionName, "\"app_name\":\"nextcloud\"") || !strings.Contains(subscriptionName, "\"container_id\":\"nextcloud-web-1\"") {
t.Fatalf("expected subscription args for nextcloud-web-1, got %q", subscriptionName)
}
writeRPCResult(t, conn, subscribeReq.ID, "sub-logs")
writeRPCNotification(t, conn, "collection_update", map[string]any{
"collection": "app.container_log_follow:{\"app_name\":\"nextcloud\",\"container_id\":\"nextcloud-web-1\",\"tail_lines\":2}",
"fields": map[string]any{
"data": "ready",
"timestamp": "2026-03-29T18:00:00Z",
},
})
writeRPCNotification(t, conn, "collection_update", map[string]any{
"collection": "app.container_log_follow:{\"app_name\":\"nextcloud\",\"container_id\":\"nextcloud-web-1\",\"tail_lines\":2}",
"fields": map[string]any{
"data": "serving",
"timestamp": "2026-03-29T18:01:00Z",
},
})
expectRPCUnsubscribe(t, conn, "sub-logs")
time.Sleep(defaultAppLogIdleWait + 100*time.Millisecond)
})
t.Cleanup(server.Close)
client := mustClientForServer(t, server.URL, ClientConfig{APIKey: "api-key"})
lines, err := client.GetAppLogs(context.Background(), "nextcloud", "nextcloud-web-1", 2)
if err != nil {
t.Fatalf("GetAppLogs() error = %v", err)
}
if len(lines) != 2 {
t.Fatalf("expected two log lines, got %+v", lines)
}
if lines[0].Timestamp != "2026-03-29T18:00:00Z" || lines[0].Data != "ready" {
t.Fatalf("unexpected first log line: %+v", lines[0])
}
if lines[1].Data != "serving" {
t.Fatalf("unexpected second log line: %+v", lines[1])
}
}
func TestGetSystemTelemetryFromRPC(t *testing.T) {
server := newMockServerWithRPC(t, defaultAPIResponses(), nil, func(t *testing.T, conn *websocket.Conn) {
authReq := readRPCRequest(t, conn)
if authReq.Method != "auth.login_with_api_key" {
t.Fatalf("expected api-key auth method, got %q", authReq.Method)
}
writeRPCResult(t, conn, authReq.ID, true)
temperatureReq := readRPCRequest(t, conn)
if temperatureReq.Method != "reporting.get_data" {
t.Fatalf("expected reporting.get_data, got %q", temperatureReq.Method)
}
writeRPCResult(t, conn, temperatureReq.ID, []map[string]any{{
"name": "cputemp",
"identifier": nil,
"legend": []string{"cpu_package", "core 0", "core 1"},
"aggregations": map[string]any{
"mean": map[string]any{
"cpu_package": 61.5,
"core 0": 58.0,
"core 1": 59.0,
},
"min": map[string]any{
"cpu_package": 60.0,
"core 0": 57.0,
"core 1": 58.0,
},
"max": map[string]any{
"cpu_package": 63.0,
"core 0": 60.0,
"core 1": 61.0,
},
},
"data": []any{},
"start": time.Now().Add(-5 * time.Minute).Unix(),
"end": time.Now().Unix(),
}})
subscribeReq := readRPCRequest(t, conn)
if subscribeReq.Method != "core.subscribe" {
t.Fatalf("expected core.subscribe, got %q", subscribeReq.Method)
}
writeRPCResult(t, conn, subscribeReq.ID, "sub-1")
writeRPCNotification(t, conn, "collection_update", map[string]any{
"collection": "reporting.realtime:{\"interval\":2}",
"fields": map[string]any{
"cpu": map[string]any{
"usage": 41,
},
"memory": map[string]any{
"physical_memory_total": 68719476736,
"physical_memory_available": 21474836480,
},
"interfaces": map[string]any{
"enp1s0": map[string]any{"rx_bytes": 4096, "tx_bytes": 2048},
"enp2s0": map[string]any{"received_bytes": 1024, "sent_bytes": 512},
},
"disks": map[string]any{
"sda": map[string]any{"read_bytes": 2048, "write_bytes": 1024},
"nvme0n1": map[string]any{"read_bytes": 4096, "write_bytes": 3072},
},
},
})
expectRPCUnsubscribe(t, conn, "sub-1")
})
t.Cleanup(server.Close)
client := mustClientForServer(t, server.URL, ClientConfig{APIKey: "api-key"})
system, err := client.GetSystemTelemetry(context.Background())
if err != nil {
t.Fatalf("GetSystemTelemetry() error = %v", err)
}
if system == nil {
t.Fatal("expected system telemetry")
}
if system.CPUPercent != 41 {
t.Fatalf("expected cpu percent 41, got %+v", system)
}
if system.MemoryTotalBytes != 68719476736 || system.MemoryAvailableBytes != 21474836480 {
t.Fatalf("unexpected memory telemetry: %+v", system)
}
if system.NetInRate != 5120 || system.NetOutRate != 2560 {
t.Fatalf("unexpected network telemetry: %+v", system)
}
if system.DiskReadRate != 6144 || system.DiskWriteRate != 4096 {
t.Fatalf("unexpected disk telemetry: %+v", system)
}
if got := system.TemperatureCelsius["cpu_package"]; got != 61.5 {
t.Fatalf("expected cpu_package temperature 61.5, got %+v", system.TemperatureCelsius)
}
if got := system.TemperatureCelsius["cpu_core_0"]; got != 58.0 {
t.Fatalf("expected cpu_core_0 temperature 58.0, got %+v", system.TemperatureCelsius)
}
if system.IntervalSeconds != 2 || system.CollectedAt.IsZero() {
t.Fatalf("expected interval/collectedAt metadata, got %+v", system)
}
}
func TestGetSystemMetricHistoryUsesReportingRPC(t *testing.T) {
server := newMockServerWithRPC(t, defaultAPIResponses(), nil, func(t *testing.T, conn *websocket.Conn) {
authReq := readRPCRequest(t, conn)
if authReq.Method != "auth.login_with_api_key" {
t.Fatalf("expected api-key auth method, got %q", authReq.Method)
}
writeRPCResult(t, conn, authReq.ID, true)
historyReq := readRPCRequest(t, conn)
if historyReq.Method != "reporting.get_data" {
t.Fatalf("expected reporting.get_data, got %q", historyReq.Method)
}
params, ok := historyReq.Params.([]any)
if !ok || len(params) != 2 {
t.Fatalf("unexpected history params: %#v", historyReq.Params)
}
graphs, ok := params[0].([]any)
if !ok || len(graphs) != 5 {
t.Fatalf("unexpected history graphs: %#v", params[0])
}
query, ok := params[1].(map[string]any)
if !ok || query["aggregate"] != true {
t.Fatalf("expected aggregate=true history query, got %#v", params[1])
}
now := time.Now().UTC().Truncate(time.Second)
writeRPCResult(t, conn, historyReq.ID, []map[string]any{
{
"name": "cpu",
"identifier": nil,
"legend": []string{"usage"},
"data": []any{
map[string]any{"timestamp": now.Add(-2 * time.Hour).Unix(), "usage": 21.0},
map[string]any{"timestamp": now.Unix(), "usage": 34.0},
},
"aggregations": map[string]any{},
"start": now.Add(-2 * time.Hour).Unix(),
"end": now.Unix(),
},
{
"name": "memory",
"identifier": nil,
"legend": []string{"used", "total"},
"data": []any{
map[string]any{"timestamp": now.Add(-2 * time.Hour).Unix(), "used": 8.0 * 1024 * 1024 * 1024, "total": 16.0 * 1024 * 1024 * 1024},
map[string]any{"timestamp": now.Unix(), "used": 10.0 * 1024 * 1024 * 1024, "total": 16.0 * 1024 * 1024 * 1024},
},
"aggregations": map[string]any{},
"start": now.Add(-2 * time.Hour).Unix(),
"end": now.Unix(),
},
{
"name": "interface",
"identifier": nil,
"legend": []string{"received", "sent"},
"data": []any{
map[string]any{"timestamp": now.Add(-2 * time.Hour).Unix(), "received": 1024.0, "sent": 512.0},
map[string]any{"timestamp": now.Unix(), "received": 4096.0, "sent": 2048.0},
},
"aggregations": map[string]any{},
"start": now.Add(-2 * time.Hour).Unix(),
"end": now.Unix(),
},
{
"name": "disk",
"identifier": nil,
"legend": []string{"read", "write"},
"data": []any{
map[string]any{"timestamp": now.Add(-2 * time.Hour).Unix(), "read": 2048.0, "write": 1024.0},
map[string]any{"timestamp": now.Unix(), "read": 8192.0, "write": 4096.0},
},
"aggregations": map[string]any{},
"start": now.Add(-2 * time.Hour).Unix(),
"end": now.Unix(),
},
})
})
t.Cleanup(server.Close)
client := mustClientForServer(t, server.URL, ClientConfig{APIKey: "api-key"})
history, err := client.GetSystemMetricHistory(context.Background(), 4*time.Hour)
if err != nil {
t.Fatalf("GetSystemMetricHistory() error = %v", err)
}
if history == nil {
t.Fatal("expected system metric history")
}
if got := len(history.CPUPercent); got != 2 {
t.Fatalf("expected cpu history, got %+v", history)
}
if got := len(history.MemoryPercent); got != 2 {
t.Fatalf("expected memory percent history, got %+v", history)
}
if got := history.MemoryPercent[1].Value; got <= 0 {
t.Fatalf("expected non-zero memory percent, got %+v", history.MemoryPercent)
}
if got := history.NetInRate[1].Value; got != 4096.0 {
t.Fatalf("expected network history, got %+v", history.NetInRate)
}
if got := history.DiskWriteRate[1].Value; got != 4096.0 {
t.Fatalf("expected disk history, got %+v", history.DiskWriteRate)
}
}
func TestGetSystemTelemetryIgnoresUnavailableTemperatureRPC(t *testing.T) {
server := newMockServerWithRPC(t, defaultAPIResponses(), nil, func(t *testing.T, conn *websocket.Conn) {
authReq := readRPCRequest(t, conn)
if authReq.Method != "auth.login_with_api_key" {
t.Fatalf("expected api-key auth method, got %q", authReq.Method)
}
writeRPCResult(t, conn, authReq.ID, true)
temperatureReq := readRPCRequest(t, conn)
if temperatureReq.Method != "reporting.get_data" {
t.Fatalf("expected reporting.get_data, got %q", temperatureReq.Method)
}
writeRPCError(t, conn, temperatureReq.ID, -32601, "not found")
subscribeReq := readRPCRequest(t, conn)
if subscribeReq.Method != "core.subscribe" {
t.Fatalf("expected core.subscribe, got %q", subscribeReq.Method)
}
writeRPCResult(t, conn, subscribeReq.ID, "sub-1")
writeRPCNotification(t, conn, "collection_update", map[string]any{
"collection": "reporting.realtime:{\"interval\":2}",
"fields": map[string]any{
"cpu": map[string]any{"usage": 41},
},
})
expectRPCUnsubscribe(t, conn, "sub-1")
})
t.Cleanup(server.Close)
client := mustClientForServer(t, server.URL, ClientConfig{APIKey: "api-key"})
system, err := client.GetSystemTelemetry(context.Background())
if err != nil {
t.Fatalf("GetSystemTelemetry() error = %v", err)
}
if system == nil {
t.Fatal("expected system telemetry")
}
if system.CPUPercent != 41 {
t.Fatalf("expected cpu percent 41, got %+v", system)
}
if len(system.TemperatureCelsius) != 0 {
t.Fatalf("expected unavailable temperature RPC to be ignored, got %+v", system.TemperatureCelsius)
}
}
func TestGetDiskTemperaturesSupportsArrayShape(t *testing.T) {
server := newMockServer(t, map[string]apiResponse{
"/api/v2.0/disk/temperatures": {
body: `[{"name":"sda","temperature":33},{"identifier":"{disk-2}","temperature_celsius":"48"},{"serial":"SER-C","temperature":{"parsed":52}}]`,
},
}, nil)
t.Cleanup(server.Close)
client := mustClientForServer(t, server.URL, ClientConfig{APIKey: "api-key"})
temperatures, err := client.GetDiskTemperatures(context.Background())
if err != nil {
t.Fatalf("GetDiskTemperatures() error = %v", err)
}
if got := temperatures["sda"]; got != 33 {
t.Fatalf("expected sda temperature 33, got %d", got)
}
if got := temperatures["{disk-2}"]; got != 48 {
t.Fatalf("expected {disk-2} temperature 48, got %d", got)
}
if got := temperatures["SER-C"]; got != 52 {
t.Fatalf("expected SER-C temperature 52, got %d", got)
}
}
func TestGetDisksToleratesUnavailableTemperatureEndpoint(t *testing.T) {
server := newMockServer(t, map[string]apiResponse{
"/api/v2.0/disk": {
body: `[{"identifier":"{disk-1}","name":"sda","serial":"SER-A","size":1000000,"model":"Seagate","type":"HDD","pool":"tank","bus":"SATA","rotationrate":7200,"status":"ONLINE"}]`,
},
"/api/v2.0/disk/temperatures": {
status: http.StatusNotFound,
body: `{"error":"not found"}`,
},
}, nil)
t.Cleanup(server.Close)
client := mustClientForServer(t, server.URL, ClientConfig{APIKey: "api-key"})
disks, err := client.GetDisks(context.Background())
if err != nil {
t.Fatalf("GetDisks() error = %v", err)
}
if len(disks) != 1 {
t.Fatalf("expected 1 disk, got %d", len(disks))
}
if disks[0].Temperature != 0 {
t.Fatalf("expected unavailable temperature to stay empty, got %+v", disks[0])
}
}
func TestGetDiskTemperaturesFallsBackToReportingRPC(t *testing.T) {
connectionCount := 0
server := newMockServerWithRPC(t, map[string]apiResponse{
"/api/v2.0/disk": {
body: `[{"identifier":"{disk-1}","name":"sda","serial":"SER-A","size":1000000,"model":"Seagate","type":"HDD","pool":"tank","bus":"SATA","rotationrate":7200,"status":"ONLINE"}]`,
},
"/api/v2.0/disk/temperatures": {
status: http.StatusNotFound,
body: `{"error":"not found"}`,
},
}, nil, func(t *testing.T, conn *websocket.Conn) {
authReq := readRPCRequest(t, conn)
if authReq.Method != "auth.login_with_api_key" {
t.Fatalf("expected api-key auth method, got %q", authReq.Method)
}
writeRPCResult(t, conn, authReq.ID, true)
connectionCount++
temperatureReq := readRPCRequest(t, conn)
if connectionCount == 1 {
if temperatureReq.Method != "disk.query" {
t.Fatalf("expected disk.query, got %q", temperatureReq.Method)
}
writeRPCResult(t, conn, temperatureReq.ID, defaultRoutePayloadMaps(t, "/api/v2.0/disk")[:1])
return
}
if temperatureReq.Method != "reporting.get_data" {
t.Fatalf("expected reporting.get_data, got %q", temperatureReq.Method)
}
params, ok := temperatureReq.Params.([]any)
if !ok || len(params) != 2 {
t.Fatalf("unexpected reporting params: %#v", temperatureReq.Params)
}
graphs, ok := params[0].([]any)
if !ok || len(graphs) != 1 {
t.Fatalf("unexpected reporting graphs: %#v", params[0])
}
graph, ok := graphs[0].(map[string]any)
if !ok {
t.Fatalf("unexpected reporting graph entry: %#v", graphs[0])
}
if got := readStringAny(graph, "name"); got != "disktemp" {
t.Fatalf("expected disktemp graph, got %q", got)
}
if got := readStringAny(graph, "identifier"); got != "sda" {
t.Fatalf("expected sda identifier, got %q", got)
}
writeRPCResult(t, conn, temperatureReq.ID, []map[string]any{{
"name": "disktemp",
"identifier": "sda",
"legend": []string{"temperature"},
"aggregations": map[string]any{
"mean": map[string]any{
"temperature": 41.8,
},
},
"data": []any{},
"start": time.Now().Add(-5 * time.Minute).Unix(),
"end": time.Now().Unix(),
}})
})
t.Cleanup(server.Close)
client := mustClientForServer(t, server.URL, ClientConfig{APIKey: "api-key"})
temperatures, err := client.GetDiskTemperatures(context.Background())
if err != nil {
t.Fatalf("GetDiskTemperatures() error = %v", err)
}
if got := temperatures["sda"]; got != 42 {
t.Fatalf("expected sda reporting fallback temperature 42, got %d", got)
}
}
func TestGetDisksFallsBackToReportingRPCWhenTemperatureEndpointUnavailable(t *testing.T) {
connectionCount := 0
server := newMockServerWithRPC(t, map[string]apiResponse{
"/api/v2.0/disk": {
body: `[{"identifier":"{disk-1}","name":"sda","serial":"SER-A","size":1000000,"model":"Seagate","type":"HDD","pool":"tank","bus":"SATA","rotationrate":7200,"status":"ONLINE"}]`,
},
"/api/v2.0/disk/temperatures": {
status: http.StatusNotFound,
body: `{"error":"not found"}`,
},
}, nil, func(t *testing.T, conn *websocket.Conn) {
authReq := readRPCRequest(t, conn)
if authReq.Method != "auth.login_with_api_key" {
t.Fatalf("expected api-key auth method, got %q", authReq.Method)
}
writeRPCResult(t, conn, authReq.ID, true)
connectionCount++
request := readRPCRequest(t, conn)
switch connectionCount {
case 1:
if request.Method != "disk.query" {
t.Fatalf("expected disk.query, got %q", request.Method)
}
writeRPCResult(t, conn, request.ID, defaultRoutePayloadMaps(t, "/api/v2.0/disk")[:1])
case 2:
if request.Method != "reporting.get_data" {
t.Fatalf("expected reporting.get_data, got %q", request.Method)
}
writeRPCResult(t, conn, request.ID, []map[string]any{{
"name": "disktemp",
"identifier": "sda",
"legend": []string{"temperature"},
"aggregations": map[string]any{
"mean": map[string]any{
"temperature": 43.2,
},
},
"data": []any{},
"start": time.Now().Add(-5 * time.Minute).Unix(),
"end": time.Now().Unix(),
}})
case 3:
if request.Method != "disk.temperature_agg" {
t.Fatalf("expected disk.temperature_agg, got %q", request.Method)
}
writeRPCResult(t, conn, request.ID, map[string]any{
"sda": map[string]any{
"min": 39.0,
"avg": 41.6,
"max": 45.0,
"window_days": 7,
},
})
default:
t.Fatalf("unexpected extra websocket connection %d", connectionCount)
}
})
t.Cleanup(server.Close)
client := mustClientForServer(t, server.URL, ClientConfig{APIKey: "api-key"})
disks, err := client.GetDisks(context.Background())
if err != nil {
t.Fatalf("GetDisks() error = %v", err)
}
if len(disks) != 1 {
t.Fatalf("expected 1 disk, got %d", len(disks))
}
if got := disks[0].Temperature; got != 43 {
t.Fatalf("expected reporting fallback temperature 43, got %+v", disks[0])
}
if got := disks[0].TemperatureAggregate.MaxCelsius; got != 45.0 {
t.Fatalf("expected aggregate max 45.0, got %+v", disks[0].TemperatureAggregate)
}
}
func TestGetDisksIncludesDiskTemperatureAggregatesFromRPC(t *testing.T) {
connectionCount := 0
server := newMockServerWithRPC(t, map[string]apiResponse{
"/api/v2.0/disk": {
body: `[{"identifier":"{disk-1}","name":"sda","serial":"SER-A","size":1000000,"model":"Seagate","type":"HDD","pool":"tank","bus":"SATA","rotationrate":7200,"status":"ONLINE"}]`,
},
"/api/v2.0/disk/temperatures": {
body: `{"sda":34}`,
},
}, nil, func(t *testing.T, conn *websocket.Conn) {
authReq := readRPCRequest(t, conn)
if authReq.Method != "auth.login_with_api_key" {
t.Fatalf("expected api-key auth method, got %q", authReq.Method)
}
writeRPCResult(t, conn, authReq.ID, true)
connectionCount++
aggregateReq := readRPCRequest(t, conn)
if connectionCount == 1 {
if aggregateReq.Method != "disk.query" {
t.Fatalf("expected disk.query, got %q", aggregateReq.Method)
}
writeRPCResult(t, conn, aggregateReq.ID, defaultRoutePayloadMaps(t, "/api/v2.0/disk")[:1])
return
}
if aggregateReq.Method == "reporting.get_data" {
// Current-temperature reporting probe: answer empty so the
// client falls back to the REST POST /disk/temperatures route.
writeRPCResult(t, conn, aggregateReq.ID, []map[string]any{})
return
}
if aggregateReq.Method != "disk.temperature_agg" {
t.Fatalf("expected disk.temperature_agg, got %q", aggregateReq.Method)
}
params, ok := aggregateReq.Params.([]any)
if !ok || len(params) != 2 {
t.Fatalf("unexpected aggregate params: %#v", aggregateReq.Params)
}
identifiers, ok := params[0].([]any)
if !ok || len(identifiers) != 1 {
t.Fatalf("unexpected aggregate identifiers: %#v", params[0])
}
if got := strings.TrimSpace(fmt.Sprint(identifiers[0])); got != "sda" {
t.Fatalf("expected sda identifier, got %q", got)
}
if got := int(readFloatAny(map[string]any{"value": params[1]}, "value")); got != defaultDiskTemperatureAggregateWindowDays {
t.Fatalf("expected window %d, got %#v", defaultDiskTemperatureAggregateWindowDays, params[1])
}
writeRPCResult(t, conn, aggregateReq.ID, map[string]any{
"sda": map[string]any{
"min": 29.0,
"avg": 32.8,
"max": 38.0,
"window_days": 7,
},
})
})
t.Cleanup(server.Close)
client := mustClientForServer(t, server.URL, ClientConfig{APIKey: "api-key"})
disks, err := client.GetDisks(context.Background())
if err != nil {
t.Fatalf("GetDisks() error = %v", err)
}
if len(disks) != 1 {
t.Fatalf("expected 1 disk, got %d", len(disks))
}
if got := disks[0].TemperatureAggregate.WindowDays; got != 7 {
t.Fatalf("expected aggregate window 7, got %+v", disks[0].TemperatureAggregate)
}
if got := disks[0].TemperatureAggregate.MinCelsius; got != 29.0 {
t.Fatalf("expected aggregate min 29.0, got %+v", disks[0].TemperatureAggregate)
}
if got := disks[0].TemperatureAggregate.AvgCelsius; got != 32.8 {
t.Fatalf("expected aggregate avg 32.8, got %+v", disks[0].TemperatureAggregate)
}
if got := disks[0].TemperatureAggregate.MaxCelsius; got != 38.0 {
t.Fatalf("expected aggregate max 38.0, got %+v", disks[0].TemperatureAggregate)
}
}
func TestGetDiskTemperatureHistoryUsesReportingRPC(t *testing.T) {
server := newMockServerWithRPC(t, defaultAPIResponses(), nil, func(t *testing.T, conn *websocket.Conn) {
authReq := readRPCRequest(t, conn)
if authReq.Method != "auth.login_with_api_key" {
t.Fatalf("expected api-key auth method, got %q", authReq.Method)
}
writeRPCResult(t, conn, authReq.ID, true)
historyReq := readRPCRequest(t, conn)
if historyReq.Method != "reporting.get_data" {
t.Fatalf("expected reporting.get_data, got %q", historyReq.Method)
}
params, ok := historyReq.Params.([]any)
if !ok || len(params) != 2 {
t.Fatalf("unexpected history params: %#v", historyReq.Params)
}
graphs, ok := params[0].([]any)
if !ok || len(graphs) != 1 {
t.Fatalf("unexpected history graphs: %#v", params[0])
}
graph, ok := graphs[0].(map[string]any)
if !ok {
t.Fatalf("unexpected history graph entry: %#v", graphs[0])
}
if got := readStringAny(graph, "name"); got != "disktemp" {
t.Fatalf("expected disktemp graph, got %q", got)
}
if got := readStringAny(graph, "identifier"); got != "sda" {
t.Fatalf("expected sda identifier, got %q", got)
}
query, ok := params[1].(map[string]any)
if !ok {
t.Fatalf("unexpected history query: %#v", params[1])
}
if aggregate := query["aggregate"]; aggregate != true {
t.Fatalf("expected aggregate=true for history query, got %#v", aggregate)
}
now := time.Now().UTC().Truncate(time.Second)
writeRPCResult(t, conn, historyReq.ID, []map[string]any{{
"name": "disktemp",
"identifier": "sda",
"legend": []string{"temperature"},
"data": []any{
[]any{now.Add(-2 * time.Hour).Unix(), 30.0},
[]any{now.Add(-1 * time.Hour).Unix(), 31.5},
[]any{now.Unix(), 33.0},
},
"aggregations": map[string]any{},
"start": now.Add(-2 * time.Hour).Unix(),
"end": now.Unix(),
}})
})
t.Cleanup(server.Close)
client := mustClientForServer(t, server.URL, ClientConfig{APIKey: "api-key"})
history, err := client.GetDiskTemperatureHistory(context.Background(), []string{"sda"}, 4*time.Hour)
if err != nil {
t.Fatalf("GetDiskTemperatureHistory() error = %v", err)
}
points, ok := history["sda"]
if !ok {
t.Fatalf("expected history for sda, got %#v", history)
}
if len(points) != 3 {
t.Fatalf("expected 3 history points, got %+v", points)
}
if points[0].Value != 30.0 || points[2].Value != 33.0 {
t.Fatalf("unexpected history values: %+v", points)
}
}
func TestClientHandlesHTTPAndDecodeErrors(t *testing.T) {
t.Run("non-2xx response", func(t *testing.T) {
server := newMockServer(t, map[string]apiResponse{
"/api/v2.0/pool": {status: http.StatusServiceUnavailable, body: `{"error":"down"}`},
}, nil)
t.Cleanup(server.Close)
client := mustClientForServer(t, server.URL, ClientConfig{APIKey: "key"})
_, err := client.GetPools(context.Background())
if err == nil {
t.Fatal("expected error from non-2xx response")
}
if !strings.Contains(err.Error(), "status=503") {
t.Fatalf("expected status code in error, got %v", err)
}
})
t.Run("malformed json", func(t *testing.T) {
server := newMockServer(t, map[string]apiResponse{
"/api/v2.0/system/info": {body: `{"hostname":`},
}, nil)
t.Cleanup(server.Close)
client := mustClientForServer(t, server.URL, ClientConfig{APIKey: "key"})
_, err := client.GetSystemInfo(context.Background())
if err == nil {
t.Fatal("expected malformed json error")
}
if !strings.Contains(err.Error(), "decode truenas response") {
t.Fatalf("unexpected decode error: %v", err)
}
})
t.Run("connection failure", func(t *testing.T) {
server := newMockServer(t, map[string]apiResponse{
"/api/v2.0/system/info": {body: `{"hostname":"nas","version":"v","buildtime":"b","uptime_seconds":1}`},
}, nil)
client := mustClientForServer(t, server.URL, ClientConfig{APIKey: "key"})
server.Close()
_, err := client.GetSystemInfo(context.Background())
if err == nil {
t.Fatal("expected connection error")
}
if !strings.Contains(err.Error(), "failed") {
t.Fatalf("unexpected connection error: %v", err)
}
})
}
func TestClientRejectsOversizedJSONResponses(t *testing.T) {
oversizedBody := fmt.Sprintf(
`{"hostname":"%s","version":"v","buildtime":"b","uptime_seconds":1}`,
strings.Repeat("a", int(maxResponseBodyBytes)),
)
server := newMockServer(t, map[string]apiResponse{
"/api/v2.0/system/info": {body: oversizedBody},
}, nil)
t.Cleanup(server.Close)
client := mustClientForServer(t, server.URL, ClientConfig{APIKey: "key"})
_, err := client.GetSystemInfo(context.Background())
if err == nil {
t.Fatal("expected oversized response error")
}
if !strings.Contains(err.Error(), fmt.Sprintf("response body exceeds %d bytes", maxResponseBodyBytes)) {
t.Fatalf("expected response size limit error, got %v", err)
}
}
func TestNewClientRejectsUnsafeURLComponents(t *testing.T) {
tests := []struct {
name string
host string
errContains string
}{
{
name: "credentials",
host: "https://admin:[email protected]",
errContains: "credentials are not supported",
},
{
name: "non-root path",
host: "https://truenas.local/api/v2.0",
errContains: "path is not supported",
},
{
name: "query",
host: "https://truenas.local?insecure=1",
errContains: "query is not supported",
},
{
name: "fragment",
host: "https://truenas.local#section",
errContains: "fragment is not supported",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, err := NewClient(ClientConfig{
Host: tt.host,
APIKey: "key",
})
if err == nil {
t.Fatalf("expected error for host %q", tt.host)
}
if !strings.Contains(err.Error(), tt.errContains) {
t.Fatalf("expected error containing %q, got %v", tt.errContains, err)
}
})
}
}
func TestClientTLSFingerprintPinning(t *testing.T) {
handler := http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
if request.URL.Path != "/api/v2.0/system/info" {
http.NotFound(writer, request)
return
}
writer.Header().Set("Content-Type", "application/json")
_, _ = writer.Write([]byte(`{"hostname":"nas","version":"TrueNAS-SCALE-24.10.2","buildtime":"b","uptime_seconds":1}`))
})
tlsServer := httptest.NewTLSServer(handler)
t.Cleanup(tlsServer.Close)
cert := tlsServer.Certificate()
fingerprintRaw := sha256.Sum256(cert.Raw)
fingerprint := withFingerprintColons(strings.ToUpper(hex.EncodeToString(fingerprintRaw[:])))
client := mustClientForServer(t, tlsServer.URL, ClientConfig{
APIKey: "key",
InsecureSkipVerify: true,
Fingerprint: "SHA256:" + fingerprint,
})
if err := client.TestConnection(context.Background()); err != nil {
t.Fatalf("expected pinning success, got %v", err)
}
badClient := mustClientForServer(t, tlsServer.URL, ClientConfig{
APIKey: "key",
InsecureSkipVerify: true,
Fingerprint: strings.Repeat("0", 64),
})
if err := badClient.TestConnection(context.Background()); err == nil {
t.Fatal("expected pinning failure")
}
}
func TestClientCloseClosesIdleConnections(t *testing.T) {
transport := &closeTrackingTransport{}
client := &Client{
httpClient: &http.Client{
Transport: transport,
},
}
client.Close()
if transport.closeCalls != 1 {
t.Fatalf("expected CloseIdleConnections to be called once, got %d", transport.closeCalls)
}
}
func TestClientCloseNilSafe(t *testing.T) {
var nilClient *Client
nilClient.Close()
(&Client{}).Close()
(&Client{httpClient: &http.Client{}}).Close()
}
func defaultAppQueryPayload(t *testing.T) []map[string]any {
t.Helper()
return defaultRoutePayloadMaps(t, "/api/v2.0/app")
}
func defaultRoutePayloadMaps(t *testing.T, route string) []map[string]any {
t.Helper()
response, ok := defaultAPIResponses()[route]
if !ok {
t.Fatalf("missing default fixture route %q", route)
}
var payload []map[string]any
if err := json.Unmarshal([]byte(response.body), &payload); err != nil {
t.Fatalf("unmarshal default fixture route %q: %v", route, err)
}
return payload
}
func assertQueryParams(t *testing.T, params any, method string) {
t.Helper()
values, ok := params.([]any)
if !ok || len(values) != 2 {
t.Fatalf("expected %s filters/options params, got %#v", method, params)
}
}
func defaultAPIResponses() map[string]apiResponse {
return map[string]apiResponse{
"/api/v2.0/system/info": {
body: `{"hostname":"truenas-main","version":"TrueNAS-SCALE-24.10.2","buildtime":"24.10.2.1","uptime_seconds":86400,"system_serial":"SER123","system_manufacturer":"iXsystems","physical_cores":16,"physmem":68719476736}`,
},
"/api/v2.0/pool": {
body: `[{"id":1,"name":"tank","status":"ONLINE","size":1000,"allocated":400,"free":600}]`,
},
"/api/v2.0/pool/dataset": {
body: `[{"id":"tank/apps","name":"tank/apps","pool":"tank","used":{"rawvalue":"12345","parsed":12345},"available":{"rawvalue":"555","parsed":555},"mountpoint":"/mnt/tank/apps","readonly":{"rawvalue":"off","parsed":false},"mounted":true}]`,
},
"/api/v2.0/disk": {
body: `[{"identifier":"{disk-1}","name":"sda","serial":"SER-A","size":1000000,"model":"Seagate","type":"HDD","pool":"tank","bus":"SATA","rotationrate":7200,"status":"ONLINE"},{"identifier":"{disk-2}","name":"nvme0n1","serial":"SER-B","size":2000000,"model":"Samsung","type":"SSD","pool":"tank","bus":"NVMe","rotationrate":0,"status":"ONLINE"}]`,
},
"/api/v2.0/disk/temperatures": {
body: `{"sda":34,"nvme0n1":"49","SER-B":51}`,
},
"/api/v2.0/alert/list": {
body: `[{"id":"a1","level":"WARNING","formatted":"Disk temp high","source":"DiskService","dismissed":false,"datetime":{"$date":1707400000000}}]`,
},
"/api/v2.0/service": {
body: `[{"id":1,"service":"smb","enable":true,"state":"RUNNING","pids":[2418,2420]},{"id":2,"service":"ssh","enable":false,"state":"STOPPED","pids":[]}]`,
},
"/api/v2.0/app": {
body: `[{"id":"nextcloud","name":"Nextcloud","state":"RUNNING","version":"1.0.3","human_version":"29.0.7","upgrade_available":true,"image_updates_available":true,"notes":"Team cloud","active_workloads":{"containers":2,"used_host_ips":["0.0.0.0"],"used_ports":[{"container_port":443,"protocol":"tcp","host_ports":[{"host_port":30443,"host_ip":"0.0.0.0"}]}],"container_details":[{"id":"nextcloud-web-1","service_name":"nextcloud","image":"docker.io/library/nextcloud:29.0.7","state":"running","port_config":[{"container_port":443,"protocol":"tcp","host_ports":[{"host_port":30443,"host_ip":"0.0.0.0"}]}],"volume_mounts":[{"source":"/mnt/tank/apps/nextcloud","destination":"/var/www/html","mode":"rw","type":"bind"}]},{"id":"nextcloud-redis-1","service_name":"redis","image":"docker.io/library/redis:7.2","state":"running","port_config":[],"volume_mounts":[{"source":"ix-nextcloud-redis","destination":"/data","mode":"rw","type":"volume"}]}],"volumes":[{"source":"/mnt/tank/apps/nextcloud","destination":"/var/www/html","mode":"rw","type":"bind"},{"source":"ix-nextcloud-redis","destination":"/data","mode":"rw","type":"volume"}],"images":["docker.io/library/nextcloud:29.0.7","docker.io/library/redis:7.2"],"networks":[{"name":"ix-nextcloud_default","id":"net-1","labels":{"com.docker.compose.project":"nextcloud"}}]}}]`,
},
"/api/v2.0/vm": {
body: `[{"id":42,"name":"windows-lab","vcpus":4,"cores":2,"threads":2,"memory":8192,"bootloader":"UEFI","autostart":true,"status":{"state":"RUNNING","pid":1234,"domain_state":"RUNNING"},"devices":[{"id":1,"attributes":{"dtype":"DISK"}},{"id":2,"attributes":{"dtype":"NIC"}}]}]`,
},
"/api/v2.0/sharing/smb": {
body: `[{"id":1,"name":"Media","path":"/mnt/tank/media","dataset":"tank/media","enabled":true,"comment":"Household media","readonly":false,"browsable":true,"access_based_share_enumeration":true,"locked":false,"audit":{"enable":true}}]`,
},
"/api/v2.0/sharing/nfs": {
body: `[{"id":2,"path":"/mnt/tank/projects","dataset":"tank/projects","comment":"Linux exports","enabled":true,"ro":true,"networks":["10.10.20.0/24"],"hosts":[],"security":["SYS"],"locked":false}]`,
},
}
}
func newMockServer(t *testing.T, responses map[string]apiResponse, assertRequest func(*testing.T, *http.Request)) *httptest.Server {
t.Helper()
return httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
if assertRequest != nil {
assertRequest(t, request)
}
response, ok := responses[request.URL.Path]
if !ok {
if request.URL.Path == "/api/v2.0/system/info" {
writer.Header().Set("Content-Type", "application/json")
_, _ = writer.Write([]byte(`{"hostname":"legacy-test","version":"TrueNAS-SCALE-24.10.2","buildtime":"24.10.2","uptime_seconds":1}`))
return
}
http.NotFound(writer, request)
return
}
status := response.status
if status == 0 {
status = http.StatusOK
}
contentType := response.contentType
if contentType == "" {
contentType = "application/json"
}
writer.Header().Set("Content-Type", contentType)
writer.WriteHeader(status)
_, _ = writer.Write([]byte(response.body))
}))
}
func newMockServerWithRPC(t *testing.T, responses map[string]apiResponse, assertRequest func(*testing.T, *http.Request), handleRPC func(*testing.T, *websocket.Conn)) *httptest.Server {
t.Helper()
upgrader := websocket.Upgrader{}
return httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
if request.URL.Path == "/api/current" && websocket.IsWebSocketUpgrade(request) {
conn, err := upgrader.Upgrade(writer, request, nil)
if err != nil {
t.Fatalf("upgrade websocket: %v", err)
}
defer func() { _ = conn.Close() }()
handleRPC(t, conn)
return
}
if assertRequest != nil {
assertRequest(t, request)
}
response, ok := responses[request.URL.Path]
if !ok {
http.NotFound(writer, request)
return
}
status := response.status
if status == 0 {
status = http.StatusOK
}
contentType := response.contentType
if contentType == "" {
contentType = "application/json"
}
writer.Header().Set("Content-Type", contentType)
writer.WriteHeader(status)
_, _ = writer.Write([]byte(response.body))
}))
}
func readRPCRequest(t *testing.T, conn *websocket.Conn) trueNASRPCRequest {
t.Helper()
var request trueNASRPCRequest
if err := conn.ReadJSON(&request); err != nil {
t.Fatalf("ReadJSON() rpc request error = %v", err)
}
return request
}
func writeRPCResult(t *testing.T, conn *websocket.Conn, id int64, result any) {
t.Helper()
raw, err := json.Marshal(result)
if err != nil {
t.Fatalf("Marshal() rpc result error = %v", err)
}
if err := conn.WriteJSON(trueNASRPCResponse{
JSONRPC: "2.0",
ID: id,
Result: raw,
}); err != nil {
t.Fatalf("WriteJSON() rpc result error = %v", err)
}
}
func writeRPCError(t *testing.T, conn *websocket.Conn, id int64, code int, message string) {
t.Helper()
if err := conn.WriteJSON(trueNASRPCResponse{
JSONRPC: "2.0",
ID: id,
Error: &trueNASRPCError{
Code: code,
Message: message,
},
}); err != nil {
t.Fatalf("WriteJSON() rpc error response = %v", err)
}
}
func writeRPCNotification(t *testing.T, conn *websocket.Conn, method string, params any) {
t.Helper()
raw, err := json.Marshal(params)
if err != nil {
t.Fatalf("Marshal() rpc notification params error = %v", err)
}
if err := conn.WriteJSON(trueNASRPCResponse{
JSONRPC: "2.0",
Method: method,
Params: raw,
}); err != nil {
t.Fatalf("WriteJSON() rpc notification error = %v", err)
}
}
func expectRPCUnsubscribe(t *testing.T, conn *websocket.Conn, subscriptionID string) {
t.Helper()
request := readRPCRequest(t, conn)
if request.Method != "core.unsubscribe" {
t.Fatalf("expected core.unsubscribe, got %q", request.Method)
}
params, ok := request.Params.([]any)
if !ok || len(params) != 1 || params[0] != subscriptionID {
t.Fatalf("core.unsubscribe params = %#v, want [%q]", request.Params, subscriptionID)
}
writeRPCResult(t, conn, request.ID, nil)
}
func mustClientForServer(t *testing.T, serverURL string, config ClientConfig) *Client {
t.Helper()
parsed, err := url.Parse(serverURL)
if err != nil {
t.Fatalf("failed to parse server URL %q: %v", serverURL, err)
}
port, err := strconv.Atoi(parsed.Port())
if err != nil {
t.Fatalf("failed to parse server port from %q: %v", serverURL, err)
}
config.Host = parsed.Hostname()
config.Port = port
config.UseHTTPS = parsed.Scheme == "https"
client, err := NewClient(config)
if err != nil {
t.Fatalf("NewClient() error = %v", err)
}
client.allowInsecureRPC = true
// The fixture owns persistent sessions too. Stop and join their keepalive
// before later tests replace the package logger to inspect close records.
t.Cleanup(client.Close)
return client
}
func withFingerprintColons(value string) string {
if len(value)%2 != 0 {
return value
}
parts := make([]string, 0, len(value)/2)
for i := 0; i < len(value); i += 2 {
parts = append(parts, value[i:i+2])
}
return strings.Join(parts, ":")
}
// #2077: on the legacy REST transport (TrueNAS CORE 13) there is no
// reporting.realtime JSON-RPC subscription, so the available-memory reading is
// never populated and the ARC-aware used figure collapses to total (100% used).
// These tests pin the REST reporting fallback and the "usage unavailable"
// guard that replaced the fabricated reading.
func legacyRESTReportingMemoryBody() string {
return `[
{
"name": "memory",
"identifier": null,
"legend": ["used", "free", "available", "total"],
"data": [
{"timestamp": 1789000000, "used": 12000000000, "free": 2000000000, "available": 20000000000, "total": 34359738368},
{"timestamp": 1789000060, "used": 13000000000, "free": 3000000000, "available": 21000000000, "total": 34359738368}
],
"start": 1789000000,
"end": 1789000060
},
{
"name": "arcsize",
"identifier": null,
"legend": ["arc_size"],
"data": [
{"timestamp": 1789000060, "arc_size": 8000000000}
],
"start": 1789000000,
"end": 1789000060
}
]`
}
func newLegacyRESTReportingClient(t *testing.T, routes alertArgsTransport) *Client {
t.Helper()
client, err := NewClient(ClientConfig{Host: "http://truenas.invalid", Port: 80, APIKey: "synthetic"})
if err != nil {
t.Fatal(err)
}
client.httpClient.Transport = routes
client.mode = TransportLegacyREST
return client
}
func TestRESTSystemTelemetryReadsReportingMemory(t *testing.T) {
routes := alertArgsTransport(defaultAPIResponses())
routes["/api/v2.0/system/info"] = apiResponse{body: `{"hostname":"core","version":"TrueNAS-13.0-U6.1","physmem":34359738368}`}
routes["/api/v2.0/reporting/get_data"] = apiResponse{body: legacyRESTReportingMemoryBody()}
client := newLegacyRESTReportingClient(t, routes)
telemetry, err := client.GetSystemTelemetry(context.Background())
if err != nil {
t.Fatalf("GetSystemTelemetry() error = %v", err)
}
if telemetry == nil {
t.Fatal("expected telemetry")
}
if telemetry.MemoryTotalBytes != 34359738368 {
t.Fatalf("MemoryTotalBytes = %d, want 34359738368", telemetry.MemoryTotalBytes)
}
if telemetry.MemoryAvailableBytes != 21000000000 {
t.Fatalf("MemoryAvailableBytes = %d, want latest sample 21000000000", telemetry.MemoryAvailableBytes)
}
if telemetry.ARCSizeBytes != 8000000000 {
t.Fatalf("ARCSizeBytes = %d, want 8000000000", telemetry.ARCSizeBytes)
}
metrics := metricsFromTrueNASSystem(*telemetry, 0, 0)
if metrics.Memory == nil || metrics.Memory.Used == nil {
t.Fatalf("expected derived memory usage, got %+v", metrics.Memory)
}
wantUsed := int64(34359738368 - 21000000000 - 8000000000)
if *metrics.Memory.Used != wantUsed {
t.Fatalf("used = %d, want %d", *metrics.Memory.Used, wantUsed)
}
if metrics.Memory.Percent >= 99 {
t.Fatalf("memory percent = %v, want a real reading below 99", metrics.Memory.Percent)
}
agent := agentDataFromTrueNASSystem("conn-core", *telemetry, nil, nil, false, "", false, "")
if agent.Memory == nil || agent.Memory.UsageUnavailable {
t.Fatalf("expected known agent memory, got %+v", agent.Memory)
}
if agent.Memory.Used != wantUsed || agent.Memory.Cache != 8000000000 {
t.Fatalf("unexpected agent memory: %+v", agent.Memory)
}
}
func TestRESTSystemMetricHistoryUsesReporting(t *testing.T) {
routes := alertArgsTransport(defaultAPIResponses())
routes["/api/v2.0/reporting/get_data"] = apiResponse{body: legacyRESTReportingMemoryBody()}
client := newLegacyRESTReportingClient(t, routes)
history, err := client.GetSystemMetricHistory(context.Background(), time.Hour)
if err != nil {
t.Fatalf("GetSystemMetricHistory() error = %v", err)
}
if history == nil {
t.Fatal("expected history")
}
if len(history.MemoryAvailableBytes) != 2 {
t.Fatalf("MemoryAvailableBytes points = %d, want 2", len(history.MemoryAvailableBytes))
}
if len(history.ARCSizeBytes) != 1 {
t.Fatalf("ARCSizeBytes points = %d, want 1", len(history.ARCSizeBytes))
}
}
func TestTrueNASMemoryWithoutAvailableIsNotReportedAsUsed(t *testing.T) {
system := SystemInfo{Hostname: "core", MemoryTotalBytes: 34359738368}
metrics := metricsFromTrueNASSystem(system, 0, 0)
if metrics.Memory != nil {
t.Fatalf("fabricated memory metric without an available reading: %+v", metrics.Memory)
}
agent := agentDataFromTrueNASSystem("conn-core", system, nil, nil, false, "", false, "")
if agent.Memory == nil {
t.Fatal("expected agent memory capacity")
}
if !agent.Memory.UsageUnavailable {
t.Fatalf("UsageUnavailable = false, want true: %+v", agent.Memory)
}
if agent.Memory.Total != 34359738368 || agent.Memory.Used != 0 {
t.Fatalf("unexpected agent memory: %+v", agent.Memory)
}
}
// This is a request-validating synthetic transport, not an appliance capture.
// Native CORE request/row schema acceptance remains a separate proof obligation.
type isolatedReportingTransport struct {
routes alertArgsTransport
calls int
status int
cancel context.CancelFunc
rejectGraph string
graphStatus int
body string
}
func (s *isolatedReportingTransport) RoundTrip(r *http.Request) (*http.Response, error) {
if r.URL.Path != "/api/v2.0/reporting/get_data" {
return s.routes.RoundTrip(r)
}
s.calls++
var request struct {
Graphs []map[string]any `json:"graphs"`
Query map[string]any `json:"reporting_query"`
}
if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
return nil, err
}
status, body := s.status, `[]`
if r.Method != http.MethodPost || len(request.Graphs) == 0 || request.Query["aggregate"] != true || request.Query["end"].(float64) <= request.Query["start"].(float64) {
return nil, fmt.Errorf("invalid reporting request")
}
if status == 0 {
status = http.StatusUnprocessableEntity
body = `{"error":"graph requires a device identifier"}`
if len(request.Graphs) == 1 {
switch request.Graphs[0]["name"] {
case "cpu":
status, body = 200, `[{"name":"cpu","legend":["user","nice","system","interrupt","idle"],"data":[[1789000060,8,0,3,1,88]]}]`
case "memory":
status, body = 200, `[{"name":"memory","legend":["active","inactive","wired","laundry","free"],"data":[[1789000060,1073741824,2147483648,11811160064,0,2147483648]]}]`
case "arcsize":
status, body = 200, `[{"name":"arcsize","legend":["size"],"data":[[1789000060,8589934592]]}]`
}
}
}
if len(request.Graphs) == 1 && request.Graphs[0]["name"] == s.rejectGraph {
status, body = s.graphStatus, `{"error":"synthetic graph failure"}`
}
if s.body != "" {
body = s.body
}
if s.cancel != nil {
s.cancel()
}
return (alertArgsTransport{r.URL.Path: apiResponse{status: status, body: body}}).RoundTrip(r)
}
func TestRESTReportingGraphFailurePreservesSnapshotTelemetry(t *testing.T) {
routes := alertArgsTransport(defaultAPIResponses())
routes["/api/v2.0/system/info"] = apiResponse{body: `{"hostname":"synthetic-core","version":"TrueNAS-13.0-U6.1","physmem":17179869184}`}
transport := &isolatedReportingTransport{routes: routes}
client := newLegacyRESTReportingClient(t, routes)
client.httpClient.Transport = transport
for cycle := 0; cycle < 2; cycle++ {
snapshot, err := client.FetchSnapshot(context.Background())
if err != nil {
t.Fatal(err)
}
metrics := metricsFromTrueNASSystem(snapshot.System, 0, 0)
if metrics.CPU == nil || metrics.CPU.Percent != 12 {
t.Fatalf("CPU lost: %+v", metrics.CPU)
}
if metrics.Memory == nil || metrics.Memory.Used == nil || *metrics.Memory.Used != 6<<30 || metrics.Memory.Percent != 37.5 {
t.Fatalf("memory lost or fabricated: %+v", metrics.Memory)
}
agent := agentDataFromTrueNASSystem("synthetic-core", snapshot.System, nil, nil, false, "", false, "")
if agent.Memory == nil || agent.Memory.UsageUnavailable || agent.Memory.Used != 6<<30 || agent.Memory.Free != 2<<30 || agent.Memory.Cache != 8<<30 {
t.Fatalf("agent memory: %+v", agent.Memory)
}
if len(snapshot.Pools) == 0 || len(snapshot.Disks) == 0 {
t.Fatal("inventory lost")
}
}
if transport.calls != 2*(1+len(legacyRESTReportingGraphs())) {
t.Fatalf("reporting calls = %d, want bounded batch + selected graphs per snapshot", transport.calls)
}
history, err := client.GetSystemMetricHistory(context.Background(), time.Hour)
if err != nil {
t.Fatal(err)
}
if history == nil || len(history.CPUPercent) != 1 || history.CPUPercent[0].Value != 12 {
t.Fatalf("CPU history lost: %+v", history)
}
memory := systemMemoryPercentHistory(history, 16<<30)
if len(memory) != 1 || memory[0].Value != 37.5 {
t.Fatalf("history must match live free/ARC memory, got %+v", memory)
}
}
func TestRESTReportingGraphFailureBoundaries(t *testing.T) {
for _, status := range []int{200, 400, 401, 403, 404, 422, 429, 500, 503} {
t.Run(strconv.Itoa(status), func(t *testing.T) {
transport := &isolatedReportingTransport{status: status}
client := newLegacyRESTReportingClient(t, nil)
client.httpClient.Transport = transport
if _, err := client.GetSystemTelemetry(context.Background()); err == nil {
t.Fatal("all failed graphs must return an error")
}
want := 1
if status == 400 || status == 422 || status == 500 {
want = 7
}
if transport.calls != want {
t.Fatalf("calls = %d, want %d", transport.calls, want)
}
})
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
transport := &isolatedReportingTransport{status: 422, cancel: cancel}
client := newLegacyRESTReportingClient(t, nil)
client.httpClient.Transport = transport
if _, err := client.GetSystemTelemetry(ctx); err == nil || transport.calls != 1 {
t.Fatalf("cancellation must stop fallback, calls=%d err=%v", transport.calls, err)
}
}
func TestReportingActiveMemoryIsNotTotalUsed(t *testing.T) {
for _, values := range []map[string]float64{
{"active": 1 << 30, "free": 2 << 30, "total": 16 << 30},
{"active": 1 << 30, "available": 2 << 30, "total": 16 << 30},
} {
percent, ok := parseSystemMemoryPercent(values)
if !ok || percent != 87.5 {
t.Fatalf("active pages treated as total used: %v, %v", percent, ok)
}
}
if _, ok := parseSystemMemoryPercent(map[string]float64{"active": 1 << 30, "total": 16 << 30}); ok {
t.Fatal("active alone cannot establish total usage")
}
}
func TestRESTReportingPartialAndMalformedResponses(t *testing.T) {
for _, status := range []int{422, 401, 429} {
transport := &isolatedReportingTransport{rejectGraph: "memory", graphStatus: status}
client := newLegacyRESTReportingClient(t, nil)
client.httpClient.Transport = transport
telemetry, err := client.GetSystemTelemetry(context.Background())
if status == 422 {
if err != nil || telemetry.CPUPercent != 12 || telemetry.MemoryAvailableBytes != 0 {
t.Fatalf("partial readings lost/fabricated: %+v %v", telemetry, err)
}
telemetry.MemoryTotalBytes = 16 << 30
if metricsFromTrueNASSystem(*telemetry, 0, 0).Memory != nil {
t.Fatal("missing memory became usage")
}
} else if err == nil || transport.calls != 3 {
t.Fatalf("terminal error after successful CPU must stop collection: status=%d calls=%d err=%v", status, transport.calls, err)
}
}
for _, body := range []string{`{`, legacyRESTReportingMemoryBody()} {
transport := &isolatedReportingTransport{status: 200, body: body}
client := newLegacyRESTReportingClient(t, nil)
client.httpClient.Transport = transport
_, err := client.GetSystemTelemetry(context.Background())
if (err != nil) != (body == "{") || transport.calls != 1 {
t.Fatalf("batch/decode result: calls=%d err=%v", transport.calls, err)
}
}
}
// nativeShapeReportingTransport models the request shape the TrueNAS 13
// middleware accepts: a graph that carries an explicit identifier key (even a
// null one) is rejected for the whole request, while a device-independent graph
// that omits the key succeeds. This is a schema-faithful synthetic transport,
// not an appliance capture; it encodes the native request shape the reporter
// supplied in #2077.
type nativeShapeReportingTransport struct {
routes alertArgsTransport
graphs []map[string]any
}
func (s *nativeShapeReportingTransport) RoundTrip(r *http.Request) (*http.Response, error) {
if r.URL.Path != "/api/v2.0/reporting/get_data" {
return s.routes.RoundTrip(r)
}
var request struct {
Graphs []map[string]any `json:"graphs"`
Query map[string]any `json:"reporting_query"`
}
if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
return nil, err
}
s.graphs = append(s.graphs, request.Graphs...)
for _, graph := range request.Graphs {
if _, present := graph["identifier"]; present {
return (alertArgsTransport{r.URL.Path: apiResponse{
status: http.StatusUnprocessableEntity,
body: `{"error":"identifier: Not a string"}`,
}}).RoundTrip(r)
}
}
body := `[
{"name":"cpu","legend":["user","idle"],"data":[[1789000060,12,88]]},
{"name":"memory","legend":["used","free","total"],"data":[[1789000060,6000000000,2000000000,16000000000]]},
{"name":"arcsize","legend":["size"],"data":[[1789000060,8000000000]]}
]`
return (alertArgsTransport{r.URL.Path: apiResponse{status: http.StatusOK, body: body}}).RoundTrip(r)
}
func TestRESTReportingRequestMatchesNativeGraphShape(t *testing.T) {
routes := alertArgsTransport(defaultAPIResponses())
routes["/api/v2.0/system/info"] = apiResponse{body: `{"hostname":"core","version":"TrueNAS-13.0-U6.1","physmem":16000000000}`}
transport := &nativeShapeReportingTransport{routes: routes}
client := newLegacyRESTReportingClient(t, routes)
client.httpClient.Transport = transport
telemetry, err := client.GetSystemTelemetry(context.Background())
if err != nil {
t.Fatalf("GetSystemTelemetry() error = %v", err)
}
if telemetry.CPUPercent != 12 || telemetry.MemoryAvailableBytes != 2000000000 || telemetry.ARCSizeBytes != 8000000000 {
t.Fatalf("native-shape telemetry lost: %+v", telemetry)
}
if len(transport.graphs) == 0 {
t.Fatal("no reporting graphs requested")
}
for _, graph := range transport.graphs {
if _, present := graph["identifier"]; present {
t.Fatalf("legacy REST graph %v sends an identifier key the native client omits", graph)
}
}
}
// These are synthetic response controls, not a capture from #2077's appliance.
// Pin the complete snapshot -> canonical row -> native History path so an
// optional graph cannot turn a missing measurement into a zero-valued series.
func TestRESTReportingObservationPresence(t *testing.T) {
previous := IsFeatureEnabled()
SetFeatureEnabled(true)
t.Cleanup(func() { SetFeatureEnabled(previous) })
for _, tc := range []struct {
name string
body string
want map[string]float64
}{
{"memory_only", `[{"name":"memory","legend":["free"],"data":[[1789000060,8]]}]`, map[string]float64{"memory": 50}},
{"arc_only", `[{"name":"arcsize","legend":["size"],"data":[[1789000060,8]]}]`, nil},
{"capacity_only", `[{"name":"memory","legend":["total"],"data":[[1789000060,16]]}]`, nil},
{"cpu_zero", `[{"name":"cpu","legend":["idle"],"data":[[1789000060,100]]}]`, map[string]float64{"cpu": 0}},
{"free_zero", `[{"name":"memory","legend":["free"],"data":[[1789000060,0]]}]`, map[string]float64{"memory": 100}},
{"network_in_zero", `[{"name":"interface","legend":["received"],"data":[[1789000060,0]]}]`, map[string]float64{"netin": 0}},
{"disk_write_zero", `[{"name":"disk","legend":["write"],"data":[[1789000060,0]]}]`, map[string]float64{"diskwrite": 0}},
{"null_array", `[{"name":"cpu","legend":["usage"],"data":[[1789000060,null]]}]`, nil},
{"null_object", `[{"name":"cpu","legend":["usage"],"data":[{"timestamp":1789000060,"usage":null}]}]`, nil},
{"malformed_object", `[{"name":"cpu","legend":["usage"],"data":[{"timestamp":1789000060,"usage":"missing","unrelated":10}]}]`, nil},
{"generic_zero", `[{"name":"cpu","legend":["usage"],"data":[{"timestamp":1789000060,"value":0}]}]`, map[string]float64{"cpu": 0}},
{"empty", `[]`, nil},
} {
t.Run(tc.name, func(t *testing.T) {
routes := alertArgsTransport(defaultAPIResponses())
routes["/api/v2.0/system/info"] = apiResponse{body: `{"hostname":"synthetic-core","version":"TrueNAS-13.0-U6.1","physmem":16,"cores":8}`}
routes["/api/v2.0/reporting/get_data"] = apiResponse{body: tc.body}
client := newLegacyRESTReportingClient(t, routes)
defer client.Close()
provider := NewLiveProviderForConnection(&APIFetcher{Client: client}, "synthetic-connection")
for cycle := 0; cycle < 2; cycle++ {
if err := provider.Refresh(context.Background()); err != nil {
t.Fatal(err)
}
snapshot := provider.Snapshot()
if snapshot.System.CPUCount != 8 || snapshot.System.MemoryTotalBytes != 16 || len(snapshot.Pools) == 0 || len(snapshot.Disks) == 0 {
t.Fatal("partial/unavailable telemetry must not discard hardware or inventory")
}
var metrics *unifiedresources.ResourceMetrics
for _, record := range provider.Records() {
if record.Resource.Type == unifiedresources.ResourceTypeAgent {
metrics = record.Resource.Metrics
}
}
if metrics == nil {
t.Fatal("missing canonical host row")
}
for key, metric := range map[string]*unifiedresources.MetricValue{
"cpu": metrics.CPU, "memory": metrics.Memory, "netin": metrics.NetIn,
"netout": metrics.NetOut, "diskread": metrics.DiskRead, "diskwrite": metrics.DiskWrite,
} {
want, present := tc.want[key]
if (metric != nil) != present || (metric != nil && metric.Value != want) {
t.Errorf("cycle %d %s: metric=%+v, want present=%v value=%v", cycle, key, metric, present, want)
}
}
id, history, err := provider.SystemMetricHistory(context.Background(), time.Hour)
if err != nil || id != "synthetic-connection" {
t.Fatalf("native History target = %q, err=%v", id, err)
}
for _, key := range []string{"cpu", "memory", "netin", "netout", "diskread", "diskwrite"} {
want, present := tc.want[key]
points := history[key]
if (len(points) > 0) != present || (len(points) > 0 && (len(points) != 1 || points[0].Value != want || points[0].Timestamp.Unix() != 1789000060)) {
t.Errorf("cycle %d %s: history=%+v, want present=%v value=%v at original timestamp", cycle, key, points, present, want)
}
}
}
})
}
}
func TestRESTSnapshotRetainsSanitizedTelemetryFailure(t *testing.T) {
for _, tc := range []struct {
name string
status int
body, category string
}{
{"no_samples", 200, `[]`, "no_samples"},
{"bad_response", 200, `{`, "invalid_response"},
{"unauthorized", 401, `provider-private-text`, "authentication"},
{"forbidden", 403, `provider-private-text`, "authentication"},
{"missing_endpoint", 404, `provider-private-text`, "unsupported_endpoint"},
{"rejected_graphs", 422, `provider-private-text`, "request_rejected"},
{"rate_limited", 429, `provider-private-text`, "rate_limited"},
{"unavailable", 503, `provider-private-text`, "http_error"},
} {
t.Run(tc.name, func(t *testing.T) {
routes := alertArgsTransport(defaultAPIResponses())
routes["/api/v2.0/system/info"] = apiResponse{body: `{"hostname":"synthetic-core","version":"TrueNAS-13.0-U6.1","physmem":16}`}
transport := &isolatedReportingTransport{routes: routes, status: tc.status, body: tc.body}
client := newLegacyRESTReportingClient(t, routes)
client.httpClient.Transport = transport
defer client.Close()
provider := NewLiveProvider(&APIFetcher{Client: client})
if err := provider.Refresh(context.Background()); err != nil {
t.Fatal(err)
}
snapshot := provider.Snapshot()
availability := snapshot.System.Telemetry
if availability == nil || availability.ErrorCategory != tc.category {
t.Fatalf("missing optional telemetry failure: %+v", availability)
}
wantStatus := tc.status
if tc.status == 200 {
wantStatus = 0
}
if availability.HTTPStatus != wantStatus || len(snapshot.Pools) == 0 || len(snapshot.Disks) == 0 {
t.Fatalf("unexpected status/inventory: %+v", snapshot)
}
serialized, err := json.Marshal(availability)
if err != nil || strings.Contains(string(serialized), "provider-private-text") || strings.Contains(string(serialized), "truenas.invalid") || strings.Contains(string(serialized), "synthetic") {
t.Fatalf("diagnostics exposed provider text: %s, err=%v", serialized, err)
}
wantCalls := 1
if tc.status == 422 {
wantCalls = 1 + len(legacyRESTReportingGraphs())
}
if transport.calls != wantCalls {
t.Fatalf("reporting calls=%d, want %d", transport.calls, wantCalls)
}
availability.ErrorCategory = "mutated"
if provider.Snapshot().System.Telemetry.ErrorCategory != tc.category {
t.Fatal("returned snapshot shares mutable telemetry diagnostics")
}
transport.status, transport.body = 200, `[{"name":"cpu","legend":["usage"],"data":[[1789000060,0]]}]`
if err := provider.Refresh(context.Background()); err != nil {
t.Fatal(err)
}
recovered := provider.Snapshot().System.Telemetry
if recovered == nil || !recovered.CPU || recovered.Memory || recovered.NetIn || recovered.NetOut || recovered.DiskRead || recovered.DiskWrite || recovered.ErrorCategory != "" || recovered.HTTPStatus != 0 {
t.Fatalf("recovery retained failure or invented readings: %+v", recovered)
}
})
}
}
func TestRealtimeObservationPresence(t *testing.T) {
for _, tc := range []struct {
name string
fields map[string]any
want SystemTelemetryAvailability
}{
{"empty", nil, SystemTelemetryAvailability{}},
{"cpu_zero", map[string]any{"cpu": map[string]any{"usage": 0}}, SystemTelemetryAvailability{CPU: true}},
{"cpu_nested_zero", map[string]any{"cpu": map[string]any{"usage": map[string]any{"percent": 0}}}, SystemTelemetryAvailability{CPU: true}},
{"cpu_scale_aggregate_zero", map[string]any{"cpu": map[string]any{"cpu": map[string]any{"usage": 0, "temp": 36}}}, SystemTelemetryAvailability{CPU: true}},
{"cpu_scale_aggregate_null", map[string]any{"cpu": map[string]any{"cpu": map[string]any{"usage": nil, "temp": 36}}}, SystemTelemetryAvailability{}},
{"cpu_null", map[string]any{"cpu": map[string]any{"usage": nil}}, SystemTelemetryAvailability{}},
{"capacity_only", map[string]any{"memory": map[string]any{"total": 16}}, SystemTelemetryAvailability{}},
{"free_zero", map[string]any{"memory": map[string]any{"total": 16, "available": 0}}, SystemTelemetryAvailability{Memory: true}},
{"free_null", map[string]any{"memory": map[string]any{"total": 16, "available": nil}}, SystemTelemetryAvailability{}},
{"network_in_zero", map[string]any{"interfaces": map[string]any{"eth0": map[string]any{"rx_bytes": 0}}}, SystemTelemetryAvailability{NetIn: true}},
{"disk_write_zero", map[string]any{"disks": map[string]any{"sda": map[string]any{"write_bytes": 0}}}, SystemTelemetryAvailability{DiskWrite: true}},
} {
t.Run(tc.name, func(t *testing.T) {
telemetry := parseSystemTelemetry(tc.fields, 2, time.Now().UTC())
if telemetry.Telemetry == nil || *telemetry.Telemetry != tc.want {
t.Fatalf("presence=%+v, want %+v", telemetry.Telemetry, tc.want)
}
system := systemInfoFromResponse(systemInfoResponse{Physmem: 16})
mergeSystemTelemetry(system, telemetry)
metrics := metricsFromTrueNASSystem(*system, 0, 0)
if (metrics.CPU != nil) != tc.want.CPU || (metrics.Memory != nil) != tc.want.Memory ||
(metrics.NetIn != nil) != tc.want.NetIn || (metrics.NetOut != nil) != tc.want.NetOut ||
(metrics.DiskRead != nil) != tc.want.DiskRead || (metrics.DiskWrite != nil) != tc.want.DiskWrite {
t.Fatalf("timestamp/interval fabricated missing siblings: %+v", metrics)
}
telemetry.Telemetry.CPU = !tc.want.CPU
if system.Telemetry.CPU != tc.want.CPU {
t.Fatal("merge retained caller-owned presence pointer")
}
})
}
}
func TestSystemTelemetryFailureCategoriesAreBounded(t *testing.T) {
for _, tc := range []struct {
err error
want string
}{
{fmt.Errorf("wrapped: %w", context.Canceled), "cancelled"},
{fmt.Errorf("wrapped: %w", context.DeadlineExceeded), "timeout"},
{&RPCAuthError{Mechanism: "provider-private-text"}, "authentication"},
{&RPCError{Message: "provider-private-text", Reason: "provider-private-text"}, "method_error"},
{fmt.Errorf("provider-private-text"), "collection_error"},
} {
failure := systemTelemetryFailure(tc.err)
if failure.ErrorCategory != tc.want || failure.HTTPStatus != 0 || failure.CPU || failure.Memory {
t.Fatalf("failure=%+v, want %s", failure, tc.want)
}
body, err := json.Marshal(failure)
if err != nil || strings.Contains(string(body), "provider-private-text") {
t.Fatalf("unbounded failure detail: %s %v", body, err)
}
}
}
func TestClientFixtureOwnsRPCSessionLifetime(t *testing.T) {
var client *Client
var session *trueNASRPCClient
// Also clean up the adverse control when the old helper is restored.
t.Cleanup(func() {
if client != nil {
client.Close()
}
})
t.Run("fixture lifetime", func(t *testing.T) {
fixture := newProtocolFixture(t, func(_ int, request trueNASRPCRequest) protocolFixtureReply {
switch request.Method {
case "auth.login_ex":
return protocolFixtureReply{result: map[string]any{"response_type": "SUCCESS"}}
case "auth.login_with_api_key":
return protocolFixtureReply{result: true}
default:
return protocolFixtureReply{result: []any{}}
}
}, nil)
client = mustClientForServer(t, fixture.server.URL, ClientConfig{APIKey: "fixture-key", InsecureSkipVerify: true})
if _, err := client.GetPools(context.Background()); err != nil {
t.Fatal(err)
}
session = client.rpc
if session == nil || session.keepaliveDone == nil {
t.Fatal("fixture did not open a maintained RPC session")
}
})
if client == nil || session == nil {
t.Fatal("fixture failed before opening its session")
}
client.rpcMu.Lock()
closed, retained := client.closed, client.rpc != nil
client.rpcMu.Unlock()
if !closed || retained {
t.Fatal("fixture cleanup left its authenticated RPC client open")
}
select {
case <-session.keepaliveDone:
default:
t.Fatal("fixture cleanup did not join its session keepalive")
}
}