Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

8.8 KB
Raw Blame
package monitoring

import (
	"context"
	"crypto/sha256"
	"fmt"
	"net/http"
	"net/http/httptest"
	"sync/atomic"
	"testing"
	"time"

	"github.com/gorilla/websocket"
	"github.com/rcourtman/pulse-go-rewrite/internal/mock"
	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/internal/truenas"
	"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
	"github.com/rcourtman/pulse-go-rewrite/pkg/metrics"
)

// Only the polling clock is deterministic. Inventory/capacity is fetched by
// the real TLS client. This is not a capture from a native appliance.
type poolCapacityClockFetcher struct {
	truenas.APIFetcher
	at time.Time
}

func (f *poolCapacityClockFetcher) Fetch(ctx context.Context) (*truenas.FixtureSnapshot, error) {
	snapshot, err := f.APIFetcher.Fetch(ctx)
	if err == nil {
		snapshot.CollectedAt = f.at
	}
	return snapshot, err
}

func TestTrueNASPoolCapacityCollectionHistory(t *testing.T) {
	previous := truenas.IsFeatureEnabled()
	truenas.SetFeatureEnabled(true)
	t.Cleanup(func() { truenas.SetFeatureEnabled(previous) })
	previousMock := mock.IsMockEnabled()
	mustSetMockEnabled(t, false)
	t.Cleanup(func() { mustSetMockEnabled(t, previousMock) })

	var cycle atomic.Int32
	var poolReads atomic.Int32
	var memoryProbes atomic.Int32
	fields := []string{
		`,"size":1000,"allocated":400,"free":600`,
		`,"size":1000`, // Must not refresh the last reading as zero.
		`,"size":1000,"allocated":"unknown","free":600`,
		`,"size":1000,"allocated":1000,"free":0`,
	}
	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path == "/api/current" {
			http.NotFound(w, r)
			return
		}
		if r.URL.Path == "/websocket" {
			// This REST-only fixture has no optional CORE memory stream. DDP
			// authenticates after upgrading, not with a REST bearer header.
			// Its absence must not bypass any REST credential assertion or
			// prevent independent pool collection and History below.
			memoryProbes.Add(1)
			if r.Method != http.MethodGet || !websocket.IsWebSocketUpgrade(r) {
				t.Error("CORE memory probe did not request a WebSocket upgrade")
			}
			if r.Header.Get("Authorization") != "" {
				t.Error("REST credentials sent to an unsupported CORE memory endpoint")
			}
			http.NotFound(w, r)
			return
		}
		if r.Header.Get("Authorization") != "Bearer synthetic-a" && r.Header.Get("Authorization") != "Bearer synthetic-b" {
			t.Error("collection lost its connection authentication")
			w.WriteHeader(http.StatusUnauthorized)
			return
		}
		w.Header().Set("Content-Type", "application/json")
		var body string
		switch r.URL.Path {
		case "/api/v2.0/system/info":
			body = `{"hostname":"same-host","version":"TrueNAS-13.0-U6.8","cores":4,"physmem":4096}`
		case "/api/v2.0/pool":
			poolReads.Add(1)
			current := fields[cycle.Load()]
			if r.Header.Get("Authorization") == "Bearer synthetic-b" {
				current = `,"size":2000,"allocated":500,"free":1500`
			}
			body = `[{"id":1,"name":"tank","status":"DEGRADED","scan":{"function":"SCRUB","state":"FINISHED"},"topology":{"data":[{"type":"DISK","disk":"da0","status":"DEGRADED"}]}` + current + `}]`
		case "/api/v2.0/boot/get_state":
			body = `{"name":"boot-pool","status":"ONLINE","properties":{"size":{"parsed":200},"allocated":{"parsed":50},"free":{"parsed":150}}}`
		case "/api/v2.0/pool/dataset":
			body = `[{"id":"tank/quota","name":"tank/quota","pool":"tank","used":{"parsed":30},"available":{"parsed":70}}]`
		case "/api/v2.0/disk":
			body = `[{"identifier":"disk-1","name":"da0","size":10000,"rotationrate":"unknown","type":"UNKNOWN"}]`
		case "/api/v2.0/alert/list", "/api/v2.0/service", "/api/v2.0/app", "/api/v2.0/vm", "/api/v2.0/sharing/smb", "/api/v2.0/sharing/nfs", "/api/v2.0/zfs/snapshot", "/api/v2.0/replication", "/api/v2.0/reporting/graphs", "/api/v2.0/reporting/get_data":
			body = `[]`
		default:
			http.NotFound(w, r)
			return
		}
		_, _ = w.Write([]byte(body))
	}))
	t.Cleanup(server.Close)
	fingerprint := fmt.Sprintf("%x", sha256.Sum256(server.Certificate().Raw))
	start := time.Now().UTC().Truncate(time.Second).Add(-5 * time.Minute)
	providers := make([]*truenas.Provider, 2)
	fetchers := make([]*poolCapacityClockFetcher, 2)
	for i, name := range []string{"a", "b"} {
		// Trust only this synthetic certificate's pin. The self-signed lab
		// certificate is not a host trust-store or production TLS change.
		client, err := truenas.NewClient(truenas.ClientConfig{Host: server.URL, UseHTTPS: true, InsecureSkipVerify: true, Fingerprint: fingerprint, APIKey: "synthetic-" + name})
		if err != nil {
			t.Fatal(err)
		}
		t.Cleanup(client.Close)
		fetchers[i] = &poolCapacityClockFetcher{APIFetcher: truenas.APIFetcher{Client: client}, at: start}
		providers[i] = truenas.NewLiveProviderForConnection(fetchers[i], "connection-"+name)
	}
	persistent, err := metrics.NewStore(metrics.DefaultConfig(t.TempDir()))
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { _ = persistent.Close() })
	adapter := unifiedresources.NewMonitorAdapter(nil)
	monitor := &Monitor{resourceStore: adapter, metricsHistory: NewMetricsHistory(100, time.Hour), metricsStore: persistent}
	hostA := unifiedresources.SourceSpecificID(unifiedresources.ResourceTypeAgent, unifiedresources.SourceTrueNAS, "system:connection-a")
	hostB := unifiedresources.SourceSpecificID(unifiedresources.ResourceTypeAgent, unifiedresources.SourceTrueNAS, "system:connection-b")
	targets := map[string]string{}
	for step := range fields {
		cycle.Store(int32(step))
		var records []unifiedresources.IngestRecord
		for i, provider := range providers {
			fetchers[i].at = start.Add(time.Duration(step) * time.Minute)
			if err := provider.Refresh(context.Background()); err != nil {
				t.Fatalf("capacity metadata aborted collection: %v", err)
			}
			records = append(records, provider.Records()...)
		}
		adapter.PopulateSnapshotAndSupplemental(models.StateSnapshot{}, map[unifiedresources.DataSource][]unifiedresources.IngestRecord{unifiedresources.SourceTrueNAS: records})
		hostAChecked := false
		for _, resource := range adapter.GetAll() {
			if resource.Storage != nil && resource.Name == "tank" {
				if resource.ParentID == nil {
					t.Fatal("pool lost its owning connection")
				}
				targets[*resource.ParentID] = resource.ID
				if resource.Status != unifiedresources.StatusWarning || resource.Storage.ZFSPool == nil || resource.Storage.ZFSPool.ScanDetails == nil {
					t.Fatal("capacity availability lost pool health/scan")
				}
			}
			if resource.Type == unifiedresources.ResourceTypeAgent && resource.ID == hostA {
				hostAChecked = true
				if (resource.Metrics.Disk != nil) != (step == 0 || step == 3) {
					t.Fatal("known boot pool was presented as whole-host capacity")
				}
			}
		}
		if !hostAChecked || targets[hostA] == "" || targets[hostB] == "" || targets[hostA] == targets[hostB] {
			t.Fatal("host availability and two independent pool identities were not exercised")
		}
		monitor.syncUnifiedStorageMetrics(adapter)
	}
	if poolReads.Load() != 8 || len(targets) != 2 {
		t.Fatalf("reads=%d independent pool identities=%d", poolReads.Load(), len(targets))
	}
	if memoryProbes.Load() != 8 {
		t.Fatalf("CORE memory probes=%d, want one per connection and collection", memoryProbes.Load())
	}
	persistent.Flush()
	for _, resource := range adapter.GetAll() {
		if resource.Storage == nil {
			continue
		}
		wantValues := []float64{40, 100}
		wantTimes := []time.Time{start, start.Add(3 * time.Minute)}
		if resource.Name == "tank/quota" {
			wantValues = []float64{30, 30, 30, 30}
			wantTimes = []time.Time{start, start.Add(time.Minute), start.Add(2 * time.Minute), start.Add(3 * time.Minute)}
		} else if resource.Name == "boot-pool" {
			continue
		} else if resource.ParentID != nil && *resource.ParentID == hostB {
			wantValues = []float64{25, 25, 25, 25}
			wantTimes = []time.Time{start, start.Add(time.Minute), start.Add(2 * time.Minute), start.Add(3 * time.Minute)}
		}
		// Metrics target resolution, not a guessed cross-host pool name, owns
		// both the memory series and persistent History identity.
		target := adapter.MetricsTargetForResource(resource.ID)
		if target == nil || target.ResourceType != "storage" {
			t.Fatal("pool/dataset lost its storage target")
		}
		memory := monitor.GetStorageMetrics(target.ResourceID, time.Hour)["usage"]
		stored, err := persistent.Query("storage", target.ResourceID, "usage", start.Add(-time.Second), time.Now(), 0)
		if err != nil || len(memory) != len(wantValues) || len(stored) != len(wantValues) {
			t.Fatalf("%s History points memory=%d persisted=%d want=%d err=%v", resource.ID, len(memory), len(stored), len(wantValues), err)
		}
		for i, value := range wantValues {
			if memory[i].Value != value || stored[i].Value != value || !memory[i].Timestamp.Equal(wantTimes[i]) || !stored[i].Timestamp.Equal(wantTimes[i]) {
				t.Fatal("unknown capacity became zero, stale refresh, wrong units, timestamp or connection")
			}
		}
	}
}