Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

9.2 KB
Raw Blame
package monitoring

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"net/http/httptest"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
	"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
)

func TestGuestMemoryLifecycleHTTPCancellationPreservesSharedFence(t *testing.T) {
	for _, kind := range []string{"client", "cluster-client"} {
		for _, disposition := range []string{"guest", "connection", "unavailable", "stopped"} {
			t.Run(kind+"/"+disposition, func(t *testing.T) {
				entered, release := make(chan struct{}, 1), make(chan struct{})
				var once sync.Once
				unblock := func() { once.Do(func() { close(release) }) }
				var mu sync.Mutex
				var commands []string
				server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
					w.Header().Set("Content-Type", "application/json")
					switch {
					case strings.HasSuffix(r.URL.Path, "/nodes"):
						fmt.Fprint(w, `{"data":[]}`)
					case strings.HasSuffix(r.URL.Path, "/config"):
						fmt.Fprint(w, `{"data":{}}`)
					default:
						mu.Lock()
						commands = append(commands, r.URL.Path)
						mu.Unlock()
						if strings.Contains(r.URL.Path, "/qemu/105/agent/") {
							// Consumed headers do not establish body/command completion.
							w.WriteHeader(http.StatusOK)
							w.(http.Flusher).Flush()
							entered <- struct{}{}
							<-release
						} else {
							fmt.Fprint(w, `{"data":{"content":"MemTotal: 8192 kB\nMemAvailable: 6144 kB\nMemFree: 1024 kB\n","truncated":false}}`)
						}
					}
				}))
				defer server.Close()
				defer unblock()
				client := windowsBootstrapClient(t, server.URL, kind)
				m := &Monitor{}
				done := make(chan guestMemoryReply, 1)
				go func() {
					info, err := m.getVMAgentMemoryAvailability(context.Background(), client, "http-memory", "node", 105)
					done <- guestMemoryReply{info, err}
				}()
				select {
				case <-entered:
				case <-time.After(5 * time.Second):
					t.Fatal("synthetic file-read body did not start")
				}
				invalidateMemoryLifecycle(m, "http-memory", disposition)
				if got := readGuestMemoryReply(t, done); !errors.Is(got.err, proxmox.ErrGuestAgentDeferred) || got.info != (proxmox.LinuxMemoryAvailability{}) {
					t.Fatal("cancelled HTTP read published successful/failed guest memory")
				}
				// A new client and the replacement monitor lifecycle must both
				// respect the original consumed command's transport uncertainty.
				diagnostic := windowsBootstrapClient(t, server.URL, "client")
				if _, err := diagnostic.GetVMAgentInfo(context.Background(), "node", 105); proxmox.GuestAgentDeferredReason(err) != "agent-cooldown" {
					t.Errorf("invalidation erased shared diagnostic uncertainty: %v", err)
				}
				if _, err := m.getVMAgentMemoryAvailability(context.Background(), client, "http-memory", "node", 105); !errors.Is(err, proxmox.ErrGuestAgentDeferred) {
					t.Error("replacement memory lifecycle bypassed shared uncertainty")
				}
				if len(m.vmAgentMemCache) != 0 || len(m.vmAgentMemInFlight) != 0 {
					t.Error("cancelled/deferred memory retained a negative sample or leaked ownership")
				}
				mu.Lock()
				before := append([]string(nil), commands...)
				mu.Unlock()
				if len(before) != 1 || !strings.HasSuffix(before[0], "/file-read") {
					t.Fatal("invalidation/cooldown issued another command")
				}
				if info, err := m.getVMAgentMemoryAvailability(context.Background(), client, "http-memory", "node", 106); err != nil || info.EffectiveAvailable != 6144*1024 {
					t.Errorf("cancelled read blocked independent VM: %+v / %v", info, err)
				}
				mu.Lock()
				if len(commands) != 2 || !strings.Contains(commands[1], "/qemu/106/agent/file-read") {
					t.Error("independent VM added traffic to the cancelled VM")
				}
				mu.Unlock()
			})
		}
	}
}

func TestGuestMemoryLifecycleRestartPublishesNewSample(t *testing.T) {
	for _, kind := range []string{"client", "cluster-client"} {
		for _, collector := range []string{"cluster", "node"} {
			for _, disposition := range []string{"guest", "connection", "unavailable", "stopped"} {
				t.Run(kind+"/"+collector+"/"+disposition, func(t *testing.T) {
					f, server := newWindowsBootstrapFixture(t)
					f.osName = "Debian GNU/Linux"
					client := windowsBootstrapClient(t, server.URL, kind)
					m := windowsBootstrapMonitor(t)
					initial := windowsBootstrapPoll(t, m, client, collector, "bootstrap", "node", 105, 1)
					key := guestMemoryCacheKey("bootstrap", "node", 105)
					original := m.vmAgentMemCache[key]
					if initial.Memory.Used != int64(2*bootstrapMiB) || initial.Memory.Observation.Source != "guest-agent-meminfo" || original.fetchedAt.IsZero() {
						t.Fatal("initial real-client Linux memory sample missing")
					}
					invalidateMemoryLifecycle(m, "bootstrap", disposition)
					f.mu.Lock()
					f.linuxAvailable = 4 * bootstrapMiB
					f.mu.Unlock()
					f.poll.Store(2)
					resumed := windowsBootstrapPoll(t, m, client, collector, "bootstrap", "node", 105, 2)
					if resumed.ID != initial.ID || resumed.Memory.Used != int64(4*bootstrapMiB) || resumed.Memory.Usage != 50 || resumed.Memory.Observation.State != "current" || resumed.Memory.Observation.Source != "guest-agent-meminfo" || !resumed.Memory.Observation.ObservedAt.After(original.fetchedAt) {
						t.Errorf("restart reused pre-transition memory: %+v", resumed.Memory)
					}
					var fileReads int
					for _, command := range f.trace() {
						if command == "file-read" {
							fileReads++
						}
					}
					if fileReads != 2 {
						t.Errorf("restart file reads = %d, want two separate lifecycle samples", fileReads)
					}
					front := m.buildBroadcastFrontendStateFromSnapshot(models.StateSnapshot{VMs: []models.VM{resumed}}, m.mockModeFence.begin())
					data, err := json.Marshal(front.Resources)
					if err != nil {
						t.Fatal(err)
					}
					var served []struct{ Proxmox unifiedresources.ProxmoxData }
					if err := json.Unmarshal(data, &served); err != nil || len(served) != 1 || served[0].Proxmox.Memory == nil {
						t.Fatalf("served memory missing: %s / %v", data, err)
					}
					wire := served[0].Proxmox.Memory
					if wire.Total != resumed.Memory.Total || wire.Used != resumed.Memory.Used || wire.Free != resumed.Memory.Free || wire.Cache != resumed.Memory.Cache || wire.Usage != resumed.Memory.Usage || wire.UsageUnavailable != resumed.Memory.UsageUnavailable {
						t.Errorf("served memory lost the numeric/availability tuple: %+v", wire)
					}
					// JSON preserves the instant, not Go's process-local monotonic
					// clock component. Compare the exact wall-clock origin by Equal.
					assertGuestMemoryObservation(t, *wire, "current", "guest-agent-meminfo", resumed.Memory.Observation.ObservedAt)
					points := m.metricsHistory.GetGuestMetrics(initial.ID, "memory", time.Hour)
					if len(points) != 2 || points[0].Value != 25 || points[1].Value != 50 {
						t.Errorf("restart History retained old numeric memory: %+v", points)
					}
				})
			}
		}
	}
}

func TestGuestMemoryLifecycleCancelledRefreshPreservesOriginalSample(t *testing.T) {
	for _, outcome := range []string{"success", "failure", "empty"} {
		t.Run(outcome, func(t *testing.T) {
			key := guestMemoryCacheKey("cancel-refresh", "node", 105)
			original := agentMemCacheEntry{info: memoryLifecycleSample(4096), fetchedAt: time.Now().Add(-2 * vmAgentMemCacheTTL)}
			m := &Monitor{vmAgentMemCache: map[string]agentMemCacheEntry{key: original}}
			ctx, cancel := context.WithCancel(context.Background())
			defer cancel()
			old := &blockedGuestMemoryClient{stubPVEClient: &stubPVEClient{}, entered: make(chan context.Context, 1), release: make(chan struct{}), info: memoryLifecycleSample(6144)}
			if outcome != "success" {
				old.info = proxmox.LinuxMemoryAvailability{}
			}
			if outcome == "failure" {
				old.err = context.Canceled
			}
			var once sync.Once
			unblock := func() { once.Do(func() { close(old.release) }) }
			defer unblock()
			done := make(chan guestMemoryReply, 1)
			go func() {
				info, err := m.getVMAgentMemoryAvailability(ctx, old, "cancel-refresh", "node", 105)
				done <- guestMemoryReply{info, err}
			}()
			select {
			case <-old.entered:
			case <-time.After(5 * time.Second):
				t.Fatal("cancelled refresh did not start")
			}
			cancel()
			nextInfo := memoryLifecycleSample(5120)
			next := &guestMemoryAgentTestClient{stubPVEClient: &stubPVEClient{}, memInfo: &nextInfo}
			if _, err := m.getVMAgentMemoryAvailability(context.Background(), next, "cancel-refresh", "node", 105); !errors.Is(err, proxmox.ErrGuestAgentDeferred) || next.memCalls != 0 {
				t.Error("parent cancellation released an unfinished memory read")
			}
			unblock()
			if got := readGuestMemoryReply(t, done); !errors.Is(got.err, proxmox.ErrGuestAgentDeferred) || got.info != (proxmox.LinuxMemoryAvailability{}) {
				t.Error("cancelled refresh returned an unsupported new observation")
			}
			if m.vmAgentMemCache[key] != original || len(m.vmAgentMemInFlight) != 0 {
				t.Error("caller cancellation renewed/erased the original sample or created backoff")
			}
			if got, err := m.getVMAgentMemoryAvailability(context.Background(), next, "cancel-refresh", "node", 105); err != nil || got != nextInfo || next.memCalls != 1 {
				t.Error("ordinary post-cancellation refresh did not obtain new evidence")
			}
		})
	}
}