Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

13.4 KB
Raw Blame
package proxmox

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"net/http/httptest"
	"strings"
	"sync/atomic"
	"testing"
	"time"
)

func TestGetNodeStatusPermissionRefusalDoesNotInventMemory(t *testing.T) {
	var requests atomic.Int32
	var allowed atomic.Bool
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		requests.Add(1)
		if r.Method != http.MethodGet || r.URL.Path != "/api2/json/nodes/fixture/status" || r.Header.Get("Authorization") != "PVEAPIToken=fixture@pve!reader=fixture-token" {
			t.Error("ordinary read changed target, credential or method")
		}
		if !allowed.Load() {
			w.WriteHeader(http.StatusForbidden)
			fmt.Fprint(w, "private-fixture-response")
			return
		}
		fmt.Fprint(w, `{"data":{"cpu":0.25,"memory":{"total":256,"used":64,"free":192}}}`)
	}))
	defer server.Close()
	client, err := NewClient(ClientConfig{Host: server.URL, TokenName: "fixture@pve!reader", TokenValue: "fixture-token"})
	if err != nil {
		t.Fatal(err)
	}
	reading, err := client.GetNodeStatus(context.Background(), "fixture")
	if status, ok := APIErrorStatus(err); reading != nil || !ok || status != http.StatusForbidden || GuestAgentErrorReason(err) != "permission-denied" || requests.Load() != 1 {
		t.Fatalf("403 became a reading/retry or lost permission classification: reading=%+v err=%v requests=%d", reading, err, requests.Load())
	}
	allowed.Store(true) // Only this synthetic peer's ACL outcome changes.
	reading, err = client.GetNodeStatus(context.Background(), "fixture")
	if err != nil || reading == nil || reading.Memory == nil || reading.Memory.Total != 256 || reading.Memory.Used != 64 || reading.CPU != 0.25 || requests.Load() != 2 {
		t.Fatalf("same-credential ordinary recovery lost memory/CPU: reading=%+v err=%v requests=%d", reading, err, requests.Load())
	}
}

func TestClusterResourceCounterPresenceDistinguishesZeroNullAndMissing(t *testing.T) {
	var resource ClusterResource
	if err := json.Unmarshal([]byte(`{
		"type":"qemu",
		"diskread":0,
		"diskwrite":null,
		"netin":42
	}`), &resource); err != nil {
		t.Fatal(err)
	}

	presence := resource.IOCounters.Effective()
	if !presence.DiskRead || !presence.NetworkIn {
		t.Fatalf("explicit zero/value fields were not present: %+v", presence)
	}
	if presence.DiskWrite || presence.NetworkOut {
		t.Fatalf("null/missing fields were incorrectly present: %+v", presence)
	}
}

func TestGuestStatusTypesRetainCounterPresence(t *testing.T) {
	tests := []struct {
		name string
		read func() IOCounterPresence
	}{
		{
			name: "vm listing",
			read: func() IOCounterPresence {
				var value VM
				_ = json.Unmarshal([]byte(`{"diskread":0}`), &value)
				return value.IOCounters
			},
		},
		{
			name: "lxc status",
			read: func() IOCounterPresence {
				var value Container
				_ = json.Unmarshal([]byte(`{"diskread":0}`), &value)
				return value.IOCounters
			},
		},
		{
			name: "qemu status",
			read: func() IOCounterPresence {
				var value VMStatus
				_ = json.Unmarshal([]byte(`{"diskread":0}`), &value)
				return value.IOCounters
			},
		},
	}

	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			presence := test.read().Effective()
			if !presence.DiskRead || presence.DiskWrite || presence.NetworkIn || presence.NetworkOut {
				t.Fatalf("presence = %+v, want only diskread", presence)
			}
		})
	}
}

func TestObservationStampUsesOneReceiptTimeForResponse(t *testing.T) {
	observedAt := time.Date(2026, time.July, 24, 8, 30, 0, 0, time.UTC)

	vms := []VM{{VMID: 100}, {VMID: 101}}
	stampVMObservation(vms, observedAt)
	for _, vm := range vms {
		if !vm.ObservedAt.Equal(observedAt) {
			t.Fatalf("VM %d observedAt = %v", vm.VMID, vm.ObservedAt)
		}
	}

	containers := []Container{{VMID: 200}, {VMID: 201}}
	stampContainerObservation(containers, observedAt)
	for _, container := range containers {
		if !container.ObservedAt.Equal(observedAt) {
			t.Fatalf("container %d observedAt = %v", container.VMID, container.ObservedAt)
		}
	}

	resources := []ClusterResource{{VMID: 300}, {VMID: 301}}
	stampClusterResourceObservation(resources, observedAt)
	for _, resource := range resources {
		if !resource.ObservedAt.Equal(observedAt) {
			t.Fatalf("resource %d observedAt = %v", resource.VMID, resource.ObservedAt)
		}
	}
}

func TestInternalCounterMetadataNeverChangesProxmoxWireShape(t *testing.T) {
	payload, err := json.Marshal(ClusterResource{
		VMID:       100,
		DiskRead:   0,
		IOCounters: IOCounterPresence{Explicit: true, DiskRead: true},
		ObservedAt: time.Now(),
	})
	if err != nil {
		t.Fatal(err)
	}
	var raw map[string]json.RawMessage
	if err := json.Unmarshal(payload, &raw); err != nil {
		t.Fatal(err)
	}
	for _, key := range []string{"ioCounters", "IOCounters", "observedAt", "ObservedAt"} {
		if _, ok := raw[key]; ok {
			t.Fatalf("internal counter metadata %q leaked into JSON", key)
		}
	}
}

func TestBackupLocksRetainCumulativeCounterPresence(t *testing.T) {
	payload := []byte(`{"lock":"backup","diskread":0,"diskwrite":null,"netin":42}`)
	var listing VM
	var resource ClusterResource
	var status VMStatus
	for _, dst := range []any{&listing, &resource, &status} {
		if err := json.Unmarshal(payload, dst); err != nil {
			t.Fatal(err)
		}
	}
	for name, value := range map[string]struct {
		lock     string
		presence IOCounterPresence
	}{"listing": {listing.Lock, listing.IOCounters}, "resource": {resource.Lock, resource.IOCounters}, "status": {status.Lock, status.IOCounters}} {
		t.Run(name, func(t *testing.T) {
			p := value.presence.Effective()
			if value.lock != "backup" || !p.DiskRead || !p.NetworkIn || p.DiskWrite || p.NetworkOut {
				t.Fatalf("lock decoding corrupted presence: %#v", value)
			}
		})
	}
}

func TestUnverifiedVMConfigPreservesStatusCounterObservations(t *testing.T) {
	var commands atomic.Int32
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		switch {
		case strings.HasSuffix(r.URL.Path, "/config"):
			w.Header().Set("Content-Length", "1000")
			fmt.Fprint(w, `{"data":{}}`)
		case strings.HasSuffix(r.URL.Path, "/status/current"):
			fmt.Fprint(w, `{"data":{"status":"running","cpu":0.25,"diskread":0,"diskwrite":null,"netin":42}}`)
		default:
			commands.Add(1)
			backupAgentPayload(w, r)
		}
	}))
	defer server.Close()
	c := backupTestClient(t, guestAgentFixtureEndpoint(t, server.URL))
	if _, err := c.GetVMFSInfo(context.Background(), "node", 105); GuestAgentDeferredReason(err) != "lock-unverified" {
		t.Errorf("incomplete config did not defer the guest command: %v", err)
	}
	status, err := c.GetVMStatus(context.Background(), "node", 105)
	if err != nil {
		t.Fatal(err)
	}
	p := status.IOCounters.Effective()
	if status.CPU != 0.25 || status.NetIn != 42 || status.DiskRead != 0 || status.ObservedAt.IsZero() || !p.DiskRead || !p.NetworkIn || p.DiskWrite || p.NetworkOut {
		t.Fatalf("unverified guest lock suppressed/changed current PVE counters: %+v", status)
	}
	if got := commands.Load(); got != 0 {
		t.Errorf("guest commands with incomplete config = %d, want zero", got)
	}
}

func TestGuestAgentTransportDeferralPreservesLiveCounterReceipts(t *testing.T) {
	for _, kind := range []string{"lost reply", "redirect", "server error", "gateway error"} {
		t.Run(kind, func(t *testing.T) {
			var commands, statusCalls atomic.Int32
			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				switch {
				case strings.HasSuffix(r.URL.Path, "/config"):
					fmt.Fprint(w, `{"data":{}}`)
				case strings.HasSuffix(r.URL.Path, "/status/current"):
					statusCalls.Add(1)
					fmt.Fprint(w, `{"data":{"status":"running","cpu":0.25,"diskread":0,"diskwrite":null,"netin":42}}`)
				case strings.Contains(r.URL.Path, "/agent/"):
					commands.Add(1)
					if kind == "server error" || kind == "gateway error" {
						status := http.StatusInternalServerError
						if kind == "gateway error" {
							status = http.StatusBadGateway
						}
						w.WriteHeader(status)
						fmt.Fprint(w, "upstream unavailable")
						return
					}
					if kind == "redirect" {
						http.Redirect(w, r, "/unverified/agent", http.StatusTemporaryRedirect)
						return
					}
					conn, _, err := w.(http.Hijacker).Hijack()
					if err != nil {
						t.Error(err)
						return
					}
					conn.Close()
				default:
					http.NotFound(w, r)
				}
			}))
			defer server.Close()
			client := backupTestClient(t, guestAgentFixtureEndpoint(t, server.URL))
			wantReason := "agent-timeout"
			if kind == "redirect" {
				wantReason = "agent-redirect"
			}
			if kind == "server error" || kind == "gateway error" {
				wantReason = "agent-completion-unverified"
			}
			if _, err := client.GetVMFSInfo(context.Background(), "node", 105); GuestAgentDeferredReason(err) != wantReason {
				t.Fatalf("uncertain command not deferred: %v", err)
			}
			// The QGA uncertainty is not missing or zero CPU/I/O evidence. Ordinary
			// status still has its own completed response and observation receipt.
			before := time.Now()
			status, err := client.GetVMStatus(context.Background(), "node", 105)
			after := time.Now()
			if err != nil {
				t.Fatal(err)
			}
			presence := status.IOCounters.Effective()
			if status.CPU != 0.25 || status.DiskRead != 0 || status.NetIn != 42 || !presence.DiskRead || !presence.NetworkIn || presence.DiskWrite || presence.NetworkOut || status.ObservedAt.Before(before) || status.ObservedAt.After(after) {
				t.Fatalf("guest transport deferral changed current counter presence/receipt: %+v", status)
			}
			if _, err := client.GetVMAgentInfo(context.Background(), "node", 105); GuestAgentDeferredReason(err) != "agent-cooldown" {
				t.Fatalf("status response erased command uncertainty: %v", err)
			}
			if commands.Load() != 1 || statusCalls.Load() != 1 {
				t.Fatalf("guest/status calls = %d/%d, want 1/1", commands.Load(), statusCalls.Load())
			}
		})
	}
}

// The same ordinary status decoder also preserves memory availability presence.
func TestVMStatusMemoryAvailabilityPresence(t *testing.T) {
	t.Run("round-trip", testVMMemInfoAvailabilityRoundTrip)
	t.Run("invalid-numbers", testVMMemInfoAvailabilityRejectsInvalidNumbers)
	t.Run("reused-decode", testVMMemInfoAvailabilityPresenceDoesNotLeakAcrossDecode)
}

// Complete enumeration still owns the original counter-presence and receipt-
// time semantics; scoped-empty and older LXC encodings remain compatible.
func TestGuestInventoryAcceptedScopedAndExtendedResponses(t *testing.T) {
	for _, kind := range []string{"cluster", "qemu", "lxc"} {
		for _, body := range []string{
			`{"data":[]}`, `{"data":[],"errors":null,"total":0}`,
			`{"data":[{"vmid":105,"type":"qemu","node":"member","name":"fixture","status":"stopped","pool":"department","cpu":0,"maxcpu":4,"cpus":4,"mem":0,"maxmem":1024,"disk":0,"maxdisk":4096,"diskread":0,"netin":null,"uptime":0,"template":1,"tags":"ops","lock":"backup","agent":1,"vendor-metadata":{"version":1}}],"digest":"fixture-metadata"}`,
		} {
			t.Run(kind+"/"+body, func(t *testing.T) {
				server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { fmt.Fprint(w, body) }))
				defer server.Close()
				c := backupTestClient(t, server.URL)
				defer c.httpClient.CloseIdleConnections()
				switch kind {
				case "cluster":
					got, err := c.GetClusterResources(context.Background(), "vm")
					if err != nil {
						t.Fatal(err)
					}
					if strings.Contains(body, `"vmid"`) && (len(got) != 1 || got[0].VMID != 105 || got[0].Node != "member" || got[0].MaxCPU != 4 || got[0].Pool != "department" || got[0].Lock != "backup" || got[0].Template != 1 || got[0].IOCounters.DiskRead != true || got[0].IOCounters.NetworkIn || got[0].ObservedAt.IsZero()) {
						t.Fatalf("supported cluster fields changed: %+v", got)
					}
				case "qemu":
					got, err := c.GetVMs(context.Background(), "member")
					if err != nil {
						t.Fatal(err)
					}
					if strings.Contains(body, `"vmid"`) && (len(got) != 1 || got[0].VMID != 105 || got[0].CPUs != 4 || got[0].Agent != 1 || got[0].Lock != "backup" || got[0].Pool != "department" || got[0].IOCounters.NetworkIn || got[0].ObservedAt.IsZero()) {
						t.Fatalf("supported VM fields changed: %+v", got)
					}
				case "lxc":
					got, err := c.GetContainers(context.Background(), "member")
					if err != nil {
						t.Fatal(err)
					}
					if strings.Contains(body, `"vmid"`) && (len(got) != 1 || int(got[0].VMID) != 105 || int(got[0].CPUs) != 4 || got[0].Pool != "department" || got[0].IOCounters.NetworkIn || got[0].ObservedAt.IsZero()) {
						t.Fatalf("supported container fields changed: %+v", got)
					}
				}
			})
		}
	}
	t.Run("lxc-string-identifiers", func(t *testing.T) {
		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
			fmt.Fprint(w, `{"data":[{"vmid":"205","cpus":"2","name":"container"}]}`)
		}))
		defer server.Close()
		got, err := backupTestClient(t, server.URL).GetContainers(context.Background(), "member")
		if err != nil || len(got) != 1 || int(got[0].VMID) != 205 || int(got[0].CPUs) != 2 {
			t.Fatalf("string LXC identifiers lost: %+v %v", got, err)
		}
	})
	t.Run("all-resource-types", func(t *testing.T) {
		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
			fmt.Fprint(w, `{"data":[{"id":"node/member","type":"node","node":"member"},{"id":"storage/member/local","type":"storage","node":"member"},{"id":"pool/department","type":"pool"},{"id":"future/provider","type":"future-type"}]}`)
		}))
		defer server.Close()
		got, err := backupTestClient(t, server.URL).GetClusterResources(context.Background(), "")
		if err != nil || len(got) != 4 {
			t.Fatalf("non-guest resource compatibility lost: %+v %v", got, err)
		}
	})
}