Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

13.5 KB
Raw Blame
package monitoring

import (
	"context"
	"encoding/json"
	"fmt"
	"math"
	"net/http"
	"net/http/httptest"
	"reflect"
	"strings"
	"sync/atomic"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
	"github.com/rcourtman/pulse-go-rewrite/internal/config"
	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
	"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
)

type incompleteFilesystemClient struct {
	mockPVEClientExtended
	filesystems []proxmox.VMFileSystem
}

func (c *incompleteFilesystemClient) GetVMFSInfo(context.Context, string, int) ([]proxmox.VMFileSystem, error) {
	return c.filesystems, nil
}

func TestVMFilesystemIncompleteAlternateClientCannotPublishRemainder(t *testing.T) {
	for name, bad := range map[string]proxmox.VMFileSystem{
		"overused":        {TotalBytes: 9000, UsedBytes: 9001},
		"signed-overflow": {TotalBytes: uint64(math.MaxInt64) + 1, UsedBytes: 0},
	} {
		t.Run(name, func(t *testing.T) {
			bad.Mountpoint, bad.Type, bad.Disk = "/data", "ext4", "/dev/vdb1"
			peer := proxmox.VMFileSystem{Mountpoint: "/", Type: "ext4", Disk: "/dev/vda1", TotalBytes: 1000, UsedBytes: 0}
			c := &incompleteFilesystemClient{filesystems: []proxmox.VMFileSystem{peer, bad}}
			m := &Monitor{}
			res := proxmox.ClusterResource{Node: "node", VMID: 105, MaxDisk: 10000}
			total, used, free, usage, disks, fresh, reason := m.updateVMDisksFromGuestAgentFSInfo(context.Background(), "partial", res, c, 10000, 0, -1)
			if total != 10000 || used != 0 || free != 10000 || usage != -1 || len(disks) != 0 || fresh || reason != "agent-error" {
				t.Fatalf("invalid peer became a complete current aggregate: %d/%d/%d usage=%v rows=%+v fresh=%t reason=%q", total, used, free, usage, disks, fresh, reason)
			}
			// A real subsequent complete inventory may remove the other volume.
			c.filesystems = []proxmox.VMFileSystem{peer}
			total, used, free, usage, disks, fresh, reason = m.updateVMDisksFromGuestAgentFSInfo(context.Background(), "partial", res, c, 10000, 0, -1)
			if total != 1000 || used != 0 || free != 1000 || usage != 0 || len(disks) != 1 || !fresh || reason != "" {
				t.Fatal("complete inventory removal/zero was treated as partial")
			}
		})
	}
}

// Ordinary QEMU-only collection must not turn a malformed large volume into
// recovery/removal merely because a smaller valid volume now reports zero.
// Exercise both collectors, canonical previous state, served JSON, History and
// genuine filesystem alerts. No guest freeze, command bypass or native probe.
func TestVMFilesystemIncompletePollingPreservesHistoryAndBreach(t *testing.T) {
	for _, tc := range []struct{ name, collector, badRow string }{
		{"cluster", "cluster", `{"mountpoint":"/data","type":"ext4","total-bytes":9000}`},
		{"node", "node", `{"mountpoint":"/data","type":"ext4","total-bytes":9000}`},
		{"cluster-case-mount", "cluster", `{"mountpoint":"/data","MOUNTPOINT":"/proc","type":"ext4","disk":[{"dev":"/dev/vdb1"}],"total-bytes":9000,"used-bytes":8820}`},
		{"node-case-mount", "node", `{"mountpoint":"/data","MOUNTPOINT":"/proc","type":"ext4","disk":[{"dev":"/dev/vdb1"}],"total-bytes":9000,"used-bytes":8820}`},
		{"cluster-case-type", "cluster", `{"mountpoint":"/data","type":"ext4","TYPE":"tmpfs","disk":[{"dev":"/dev/vdb1"}],"total-bytes":9000,"used-bytes":8820}`},
		{"node-case-type", "node", `{"mountpoint":"/data","type":"ext4","TYPE":"tmpfs","disk":[{"dev":"/dev/vdb1"}],"total-bytes":9000,"used-bytes":8820}`},
	} {
		t.Run(tc.name, func(t *testing.T) {
			var phase, fsCalls atomic.Int32
			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				w.Header().Set("Content-Type", "application/json")
				switch {
				case strings.HasSuffix(r.URL.Path, "/config"):
					fmt.Fprint(w, `{"data":{}}`)
				case strings.HasSuffix(r.URL.Path, "/status/current"):
					fmt.Fprint(w, `{"data":{"status":"running","agent":1,"maxmem":1000,"mem":400,"meminfo":{"total":1000,"available":600}}}`)
				case strings.HasSuffix(r.URL.Path, "/agent/get-fsinfo"):
					fsCalls.Add(1)
					rows := `{"mountpoint":"/","type":"ext4","disk":[{"dev":"/dev/vda1"}],"total-bytes":1000,"used-bytes":980},{"mountpoint":"/data","type":"ext4","disk":[{"dev":"/dev/vdb1"}],"total-bytes":9000,"used-bytes":8820}`
					switch phase.Load() {
					case 1:
						rows = `{"mountpoint":"/","type":"ext4","disk":[{"dev":"/dev/vda1"}],"total-bytes":1000,"used-bytes":0},` + tc.badRow
					case 2:
						rows = `{"mountpoint":"/","type":"ext4","disk":[{"dev":"/dev/vda1"}],"total-bytes":1000,"used-bytes":0}`
					}
					fmt.Fprintf(w, `{"data":{"result":[%s]}}`, rows)
				case strings.HasSuffix(r.URL.Path, "/agent/get-osinfo"), strings.HasSuffix(r.URL.Path, "/agent/info"):
					fmt.Fprint(w, `{"data":{"result":{}}}`)
				case strings.HasSuffix(r.URL.Path, "/agent/network-get-interfaces"):
					fmt.Fprint(w, `{"data":{"result":[]}}`)
				default:
					t.Errorf("unexpected request: %s", r.URL.Path)
					http.NotFound(w, r)
				}
			}))
			defer server.Close()
			client, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: guestPollFixtureHangGuard})
			if err != nil {
				t.Fatal(err)
			}
			m := guestHistoryObservationMonitor(t)
			m.alertManager = alerts.NewManagerWithDataDir(t.TempDir(), alerts.WithoutPersistedAlertRestore())
			t.Cleanup(m.alertManager.Stop)
			cfg := m.alertManager.GetConfig()
			cfg.Enabled = true
			cfg.GuestDefaults.Disk = &alerts.HysteresisThreshold{Trigger: 90, Clear: 85}
			cfg.MetricTimeThresholds = map[string]map[string]int{"guest": {"disk": 0}}
			m.alertManager.UpdateConfig(cfg)
			m.config, m.state, m.rateTracker = &config.Config{}, models.NewState(), NewRateTracker()
			m.guestMetadataLimiter = make(map[string]time.Time)
			m.resourceStore = unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil))
			res := proxmox.ClusterResource{Type: "qemu", Node: "node", Name: "guest", VMID: 105, Status: "running", MaxMem: 1000, Mem: 400, MaxDisk: 10000, CPU: .2}
			id := makeGuestID("partial-filesystems", "node", 105)
			publish := func(vm models.VM) {
				m.state.UpdateVMs([]models.VM{vm})
				m.updateResourceStore(m.currentStateWithScope())
			}
			build := func() models.VM {
				t.Helper()
				// A slow poll still belongs to the cycle that started before collection.
				cycleStart := time.Now()
				previous := m.previousGuestContextForInstance("partial-filesystems").vmsByID[id]
				var vm models.VM
				if tc.collector == "node" {
					vms, _ := m.pollNodeVMsWithClusterResourceBuilder(context.Background(), "partial-filesystems", "node", []proxmox.VM{{VMID: 105, Name: res.Name, Status: res.Status, MaxMem: res.MaxMem, Mem: res.Mem, MaxDisk: res.MaxDisk, CPU: res.CPU}}, client, map[string]models.VM{id: previous}, nil)
					if len(vms) != 1 {
						t.Fatal("node collector lost guest")
					}
					vm = vms[0]
				} else {
					var ok bool
					vm, _, _, _, _, ok = m.buildVMFromClusterResource(context.Background(), "partial-filesystems", res, client, id, nil, &previous)
					if !ok {
						t.Fatal("cluster builder lost guest")
					}
					m.checkGuestAlertsForVM("partial-filesystems", vm)
				}
				publish(vm)
				m.recordGuestMetrics([]models.VM{vm}, nil, cycleStart)
				view := m.currentModeReadState().VMs()[0]
				if view.DiskStatusReason() != vm.DiskStatusReason || view.DiskObservation() != vm.DiskObservation || !reflect.DeepEqual(previousVMFromView(view).Disks, vm.Disks) {
					t.Fatalf("canonical projection changed completeness, original age or mount rows: vm=%+v view=%+v", vm, previousVMFromView(view))
				}
				if vm.Disk.Usage >= 0 && (view.DiskTotal() != vm.Disk.Total || view.DiskUsed() != vm.Disk.Used || view.DiskPercent() != vm.Disk.Usage) {
					t.Fatal("canonical projection changed a known disk tuple")
				}
				// Allocated capacity without usage is not a canonical metric.
				// Check absence on the wire below, not equality with raw allocation.
				if vm.Disk.Usage < 0 && (view.DiskTotal() != 0 || view.DiskUsed() != 0 || view.DiskPercent() != 0) {
					t.Fatal("canonical projection retained expired disk metrics")
				}
				front := m.buildBroadcastFrontendStateFromSnapshot(models.StateSnapshot{VMs: []models.VM{vm}}, m.mockModeFence.begin())
				wire, err := json.Marshal(front.Resources)
				var served []struct {
					Proxmox unifiedresources.ProxmoxData
					CPU     *models.ResourceMetricFrontend
					Memory  *models.ResourceMetricFrontend
					Disk    *models.ResourceMetricFrontend
				}
				if err != nil || json.Unmarshal(wire, &served) != nil || len(served) != 1 || served[0].Proxmox.DiskStatusReason != vm.DiskStatusReason || strings.Contains(string(wire), "diskObservation") {
					t.Fatalf("served completeness differs or private observation leaked: %s / %v", wire, err)
				}
				if served[0].CPU == nil || served[0].CPU.Current != 20 || served[0].Memory == nil || served[0].Memory.Current != 40 {
					t.Fatalf("independent current CPU/memory metrics disappeared: %s", wire)
				}
				metric := served[0].Disk
				if vm.Disk.Usage < 0 {
					if metric != nil {
						t.Fatal("unavailable disk reading became a measured wire zero")
					}
				} else if metric == nil || metric.Total == nil || metric.Used == nil || *metric.Total != vm.Disk.Total || *metric.Used != vm.Disk.Used || metric.Current != vm.Disk.Usage {
					t.Fatalf("known disk tuple or genuine measured zero lost on the wire: %s", wire)
				}
				return vm
			}
			initial := build()
			if initial.Disk.Total != 10000 || initial.Disk.Used != 9800 || initial.Disk.Usage != 98 || len(initial.Disks) != 2 || initial.DiskStatusReason != "" || initial.DiskObservation.ObservedAt.IsZero() {
				t.Fatalf("initial complete reading missing: %+v", initial)
			}
			active := m.alertManager.GetActiveAlerts()
			if len(active) != 2 {
				t.Fatalf("complete high readings did not establish two filesystem breaches: %+v", active)
			}
			breaches := make(map[string]time.Time)
			for _, alert := range active {
				breaches[alert.ID] = alert.StartTime
			}
			originalHistory := guestHistoryStoredPoints(t, m, "vm", id, "disk")
			if len(originalHistory) != 1 || originalHistory[0].Value != 98 {
				t.Fatal("complete reading did not establish History")
			}
			phase.Store(1)
			for i := 0; i < 3; i++ {
				retained := build()
				if retained.Disk != initial.Disk || !reflect.DeepEqual(retained.Disks, initial.Disks) || retained.DiskStatusReason != "prev-agent-error" || retained.DiskObservation != initial.DiskObservation {
					t.Fatalf("partial poll %d became fresh or removed an unknown volume: %+v", i, retained)
				}
			}
			aged := m.previousGuestContextForInstance("partial-filesystems").vmsByID[id]
			aged.DiskObservation.ObservedAt = time.Now().Add(-recentGuestAgentEvidenceMaxAge - time.Second)
			publish(aged)
			expired := build()
			if expired.Disk.Usage != -1 || expired.DiskStatusReason != "agent-error" || len(expired.Disks) != 0 || !expired.DiskObservation.ObservedAt.IsZero() {
				t.Fatal("repeated partial responses renewed expired complete evidence")
			}
			active = m.alertManager.GetActiveAlerts()
			if len(active) != 2 {
				t.Fatalf("partial/expired evidence cleared a genuine filesystem breach: %+v", active)
			}
			for _, alert := range active {
				if !alert.StartTime.Equal(breaches[alert.ID]) {
					t.Fatal("partial reading replaced a breach occurrence")
				}
			}
			if !reflect.DeepEqual(guestHistoryStoredPoints(t, m, "vm", id, "disk"), originalHistory) || len(m.metricsHistory.GetGuestMetrics(id, "disk", time.Hour)) != 1 || len(m.metricsHistory.GetGuestMetrics(id, "cpu", time.Hour)) != 5 || fsCalls.Load() != 5 {
				t.Fatal("partial evidence created disk History, stopped independent CPU History or retried a command")
			}
			// Complete ordinary replacement (the large volume really absent) and
			// measured zero must still remove/recover. History has second keys.
			time.Sleep(1100 * time.Millisecond)
			phase.Store(2)
			recovered := build()
			if recovered.Disk.Total != 1000 || recovered.Disk.Used != 0 || recovered.Disk.Usage != 0 || len(recovered.Disks) != 1 || recovered.DiskStatusReason != "" || !recovered.DiskObservation.ObservedAt.After(initial.DiskObservation.ObservedAt) || len(m.alertManager.GetActiveAlerts()) != 0 || fsCalls.Load() != 6 {
				t.Fatalf("complete ordinary removal/zero failed to recover: %+v", recovered)
			}
			points := guestHistoryStoredPoints(t, m, "vm", id, "disk")
			if len(points) != 2 || points[0] != originalHistory[0] || points[1].Value != 0 {
				t.Fatal("History lost high reading or failed to append genuine zero")
			}
		})
	}
}

func TestVMFilesystemIncompleteColdPollRemainsUnavailable(t *testing.T) {
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		switch {
		case strings.HasSuffix(r.URL.Path, "/config"):
			fmt.Fprint(w, `{"data":{}}`)
		case strings.HasSuffix(r.URL.Path, "/get-fsinfo"):
			fmt.Fprint(w, `{"data":{"result":[{"mountpoint":"/","type":"ext4","total-bytes":1000,"used-bytes":0},{"mountpoint":"/data","type":"ext4","total-bytes":9000}]}}`)
		default:
			http.NotFound(w, r)
		}
	}))
	defer server.Close()
	client, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: guestPollFixtureHangGuard})
	if err != nil {
		t.Fatal(err)
	}
	res := proxmox.ClusterResource{Node: "node", VMID: 105, MaxDisk: 10000}
	_, _, _, usage, disks, fresh, reason := (&Monitor{}).updateVMDisksFromGuestAgentFSInfo(context.Background(), "cold-partial", res, client, 10000, 0, -1)
	if usage != -1 || fresh || len(disks) != 0 || reason != "agent-error" {
		t.Fatalf("cold partial result became healthy usage: %v/%+v/%t/%q", usage, disks, fresh, reason)
	}
}