Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

9.8 KB
Raw Blame
package monitoring

import (
	"context"
	"fmt"
	"net/http"
	"net/http/httptest"
	"reflect"
	"strings"
	"sync/atomic"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
)

func metadataReservationClient(t *testing.T, handler http.Handler) *proxmox.Client {
	t.Helper()
	server := httptest.NewServer(handler)
	t.Cleanup(server.Close)
	client, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: 5 * time.Second})
	if err != nil {
		t.Fatal(err)
	}
	return client
}

func metadataReservationPayload(w http.ResponseWriter, r *http.Request) {
	switch {
	case strings.HasSuffix(r.URL.Path, "/config"):
		fmt.Fprint(w, `{"data":{}}`)
	case strings.HasSuffix(r.URL.Path, "/network-get-interfaces"):
		fmt.Fprint(w, `{"data":{"result":[{"name":"eth0","hardware-address":"02:00:00:00:00:01","ip-addresses":[{"ip-address":"192.0.2.20","prefix":24,"ip-address-type":"ipv4"}]}]}}`)
	case strings.HasSuffix(r.URL.Path, "/get-osinfo"):
		fmt.Fprint(w, `{"data":{"result":{"name":"FixtureOS","version":"2"}}}`)
	case strings.HasSuffix(r.URL.Path, "/info"):
		fmt.Fprint(w, `{"data":{"result":{"version":"2.0"}}}`)
	default:
		http.NotFound(w, r)
	}
}

func assertMetadataReservationReleased(t *testing.T, m *Monitor) {
	t.Helper()
	m.guestMetadataLimiterMu.Lock()
	defer m.guestMetadataLimiterMu.Unlock()
	if len(m.guestMetadataInFlight) != 0 {
		t.Fatal("completed fetch retained in-flight ownership")
	}
}

func TestGuestMetadataReservationOwnsWholeHTTPFetch(t *testing.T) {
	for _, seeded := range []bool{false, true} {
		for _, expiredHold := range []bool{false, true} {
			t.Run(fmt.Sprintf("cached-%t/hold-expired-%t", seeded, expiredHold), func(t *testing.T) {
				key := guestMetadataCacheKey("reservation", "node", 105)
				original := guestMetadataCacheEntry{}
				m := &Monitor{guestMetadataLimiter: make(map[string]time.Time), guestMetadataSlots: make(chan struct{}, 2)}
				if seeded {
					original = metadataObservationFixture(time.Now().Add(-2 * guestMetadataCacheTTL))
					original.osInfoSkip = true
					original.osInfoFailureCount = guestAgentOSInfoFailureThreshold
					m.guestMetadataCache = map[string]guestMetadataCacheEntry{key: original}
				}
				entered, release := make(chan struct{}), make(chan struct{})
				var commands atomic.Int32
				client := metadataReservationClient(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
					if strings.Contains(r.URL.Path, "/agent/") {
						commands.Add(1)
					}
					if strings.Contains(r.URL.Path, "/qemu/105/") && strings.HasSuffix(r.URL.Path, "/network-get-interfaces") {
						select {
						case <-entered:
						default:
							close(entered)
						}
						<-release
					}
					metadataReservationPayload(w, r)
				}))
				status := &proxmox.VMStatus{Agent: proxmox.VMAgentField{Value: 1}}
				done := make(chan bool, 1)
				go func() {
					_, _, _, _, _, deferred := m.fetchGuestAgentMetadata(context.Background(), client, "reservation", "node", "guest", 105, status, false)
					done <- deferred
				}()
				t.Cleanup(func() {
					select {
					case <-release:
					default:
						close(release)
					}
					select {
					case <-done:
					case <-time.After(6 * time.Second):
						t.Error("fetch did not finish during cleanup")
					}
				})
				select {
				case <-entered:
				case <-time.After(5 * time.Second):
					t.Fatal("network command did not enter")
				}
				m.guestMetadataLimiterMu.Lock()
				if expiredHold {
					// Advance only test-owned scheduling state; do not shorten the
					// Proxmox uncertainty fence or wait for a real guest timeout.
					m.guestMetadataLimiter[key] = time.Now().Add(-time.Second)
				}
				before := m.guestMetadataLimiter[key]
				m.guestMetadataLimiterMu.Unlock()
				ips, ifaces, name, version, agent, deferred := m.fetchGuestAgentMetadata(context.Background(), client, "reservation", "node", "guest", 105, status, false)
				if deferred || !reflect.DeepEqual(ips, original.ipAddresses) || !reflect.DeepEqual(ifaces, original.networkInterfaces) || name != original.osName || version != original.osVersion || agent != original.agentVersion {
					t.Error("duplicate fetch invented a transport pause or changed retained identity")
				}
				m.guestMetadataLimiterMu.Lock()
				unchanged := m.guestMetadataLimiter[key].Equal(before)
				m.guestMetadataLimiterMu.Unlock()
				if !unchanged || commands.Load() != 1 || len(m.guestMetadataSlots) != 1 {
					t.Error("duplicate fetch changed retry timing, sent a command or consumed another slot")
				}
				// A different VM is not blocked by this metadata reservation.
				_, _, _, _, otherAgent, otherDeferred := m.fetchGuestAgentMetadata(context.Background(), client, "reservation", "node", "other", 106, status, false)
				if otherDeferred || otherAgent != "2.0" {
					t.Error("reservation blocked another VM")
				}
				close(release)
				select {
				case deferred := <-done:
					if deferred {
						t.Error("owner did not complete normally")
					}
					// Leave a completed marker for the unconditional cleanup drain.
					done <- false
				case <-time.After(5 * time.Second):
					t.Fatal("owner did not finish")
				}
				assertMetadataReservationReleased(t, m)
				m.guestMetadataMu.Lock()
				entry := m.guestMetadataCache[key]
				entry.fetchedAt = time.Now().Add(-2 * guestMetadataCacheTTL)
				m.guestMetadataCache[key] = entry
				m.guestMetadataMu.Unlock()
				m.guestMetadataLimiterMu.Lock()
				m.guestMetadataLimiter[key] = time.Now().Add(-time.Second)
				m.guestMetadataLimiterMu.Unlock()
				_, _, _, _, agent, deferred = m.fetchGuestAgentMetadata(context.Background(), client, "reservation", "node", "guest", 105, status, false)
				if deferred || agent != "2.0" || !m.hasRecentGuestMetadataEvidence("reservation", "node", 105, time.Now()) || len(m.guestMetadataSlots) != 0 {
					t.Error("completed owner did not allow ordinary refresh")
				}
				assertMetadataReservationReleased(t, m)
			})
		}
	}
}

func TestGuestMetadataColdCacheHonoursEarlyBackoff(t *testing.T) {
	for _, mode := range []string{"unverified-lock", "cancelled-slot"} {
		t.Run(mode, func(t *testing.T) {
			key := guestMetadataCacheKey("backoff", "node", 105)
			m := &Monitor{guestMetadataLimiter: make(map[string]time.Time), guestMetadataRetryBackoff: time.Minute}
			var requests atomic.Int32
			var healthy atomic.Bool
			client := metadataReservationClient(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				requests.Add(1)
				if !healthy.Load() && strings.HasSuffix(r.URL.Path, "/config") {
					fmt.Fprint(w, `{"data":null}`)
					return
				}
				metadataReservationPayload(w, r)
			}))
			status := &proxmox.VMStatus{Agent: proxmox.VMAgentField{Value: 1}}
			ctx := context.Background()
			if mode == "cancelled-slot" {
				m.guestMetadataSlots = make(chan struct{}, 1)
				m.guestMetadataSlots <- struct{}{}
				cancelled, cancel := context.WithCancel(ctx)
				cancel()
				ctx = cancelled
			}
			_, _, _, _, _, deferred := m.fetchGuestAgentMetadata(ctx, client, "backoff", "node", "guest", 105, status, false)
			if deferred != (mode == "unverified-lock") || len(m.guestMetadataCache) != 0 {
				t.Fatal("early exit fabricated metadata or lost observed deferral")
			}
			assertMetadataReservationReleased(t, m)
			if mode == "cancelled-slot" {
				<-m.guestMetadataSlots
			}
			before, next := requests.Load(), m.guestMetadataLimiter[key]
			ips, ifaces, name, version, agent, deferred := m.fetchGuestAgentMetadata(context.Background(), client, "backoff", "node", "guest", 105, status, false)
			if requests.Load() != before || !m.guestMetadataLimiter[key].Equal(next) || len(m.guestMetadataCache) != 0 || len(ips)+len(ifaces) != 0 || name+version+agent != "" || deferred {
				t.Error("cold cache bypassed retry backoff or invented guest evidence")
			}
			// No metadata from another installation can satisfy this key.
			m.guestMetadataCache = map[string]guestMetadataCacheEntry{guestMetadataCacheKey("independent", "node", 105): {osName: "independent"}}
			healthy.Store(true)
			m.guestMetadataLimiter[key] = time.Now().Add(-time.Second)
			_, _, _, _, agent, deferred = m.fetchGuestAgentMetadata(context.Background(), client, "backoff", "node", "guest", 105, status, false)
			if deferred || agent != "2.0" || !m.hasRecentGuestMetadataEvidence("backoff", "node", 105, time.Now()) || !reflect.DeepEqual(m.guestMetadataCache[guestMetadataCacheKey("independent", "node", 105)], guestMetadataCacheEntry{osName: "independent"}) {
				t.Error("healthy cold cache did not resume independently after backoff")
			}
			assertMetadataReservationReleased(t, m)
		})
	}
}

func TestGuestMetadataInvalidationKeepsActiveOwnership(t *testing.T) {
	key := guestMetadataCacheKey("reservation", "node", 105)
	other := guestMetadataCacheKey("independent", "node", 105)
	future := time.Now().Add(time.Hour)
	m := &Monitor{guestMetadataCache: map[string]guestMetadataCacheEntry{key: {osName: "old"}, other: {osName: "independent"}}, guestMetadataLimiter: map[string]time.Time{other: future}}
	if !m.tryReserveGuestMetadataFetch(key, time.Now()) {
		t.Fatal("owner could not reserve metadata")
	}
	m.clearGuestMetadataCache("reservation", "node", 105)
	if _, exists := m.guestMetadataCache[key]; exists {
		t.Error("invalidated metadata was retained")
	}
	if _, exists := m.guestMetadataLimiter[key]; exists {
		t.Error("obsolete scheduling deadline survived invalidation")
	}
	if m.tryReserveGuestMetadataFetch(key, time.Now()) {
		t.Fatal("invalidation admitted another in-flight metadata writer")
	}
	if m.guestMetadataCache[other].osName != "independent" || !m.guestMetadataLimiter[other].Equal(future) {
		t.Error("invalidation crossed installation boundaries")
	}
	m.releaseGuestMetadataFetch(key)
	assertMetadataReservationReleased(t, m)
	if !m.tryReserveGuestMetadataFetch(key, time.Now()) {
		t.Error("invalidated metadata did not allow a fresh lifecycle after completion")
	}
	m.releaseGuestMetadataFetch(key)
}