Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

9.3 KB
Raw Blame
package monitoring

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"net/http/httptest"
	"reflect"
	"strings"
	"sync/atomic"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/config"
	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
)

func testGuestAgentTransportDeferralKeepsLastKnownHistory(t *testing.T, kinds ...string) {
	for _, kind := range kinds {
		t.Run(kind, func(t *testing.T) {
			preserveOrigins := strings.HasPrefix(kind, "typed ") || strings.HasPrefix(kind, "foreign ")
			const mib = uint64(1024 * 1024)
			var phase, calls, redirected atomic.Int32
			var lost atomic.Bool
			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				switch {
				case strings.HasSuffix(r.URL.Path, "/config"):
					fmt.Fprint(w, `{"data":{}}`)
				case strings.HasSuffix(r.URL.Path, "/status/current"):
					fmt.Fprintf(w, `{"data":{"status":"running","agent":1,"maxmem":%d,"mem":%d}}`, 8*mib, 8*mib)
				case strings.Contains(r.URL.Path, "/agent/"):
					calls.Add(1)
					if r.URL.Query().Get("redirect") == "1" {
						redirected.Add(1)
					}
					if phase.Load() == 1 && lost.CompareAndSwap(false, true) {
						switch kind {
						case "malformed success":
							fmt.Fprint(w, `{"data":{"result":`)
							return
						case "ambiguous success":
							fmt.Fprint(w, `{"data":null,"data":{"result":[]}}`)
							return
						case "trailing success":
							fmt.Fprint(w, `{"data":{"result":[]}} {}`)
							return
						case "typed result":
							fmt.Fprint(w, `{"data":{"result":"wrong command id"}}`)
							return
						case "typed content":
							fmt.Fprint(w, `{"data":{"result":[],"content":false}}`)
							return
						case "typed truncated":
							fmt.Fprint(w, `{"data":{"result":[],"truncated":"false"}}`)
							return
						case "foreign file":
							fmt.Fprint(w, `{"data":{"content":"MemTotal: 8192 kB\nMemAvailable: 8192 kB\n","truncated":false}}`)
							return
						case "foreign network":
							fmt.Fprint(w, `{"data":{"result":[{"name":"eth0","ip-addresses":[]}]}}`)
							return
						case "foreign version":
							fmt.Fprint(w, `{"data":{"result":{"version":"1.0","supported_commands":[]}}}`)
							return
						}
						if status := map[string]int{
							"server error":           http.StatusInternalServerError,
							"gateway error":          http.StatusBadGateway,
							"conflict response":      http.StatusConflict,
							"too early response":     http.StatusTooEarly,
							"client closed response": 499,
						}[kind]; status != 0 {
							w.WriteHeader(status)
							fmt.Fprint(w, "upstream unavailable")
							return
						}
						if kind == "redirect" {
							http.Redirect(w, r, r.URL.Path+"?redirect=1", http.StatusTemporaryRedirect)
							return
						}
						conn, _, err := w.(http.Hijacker).Hijack()
						if err != nil {
							t.Error(err)
							return
						}
						conn.Close()
						return
					}
					switch {
					case strings.HasSuffix(r.URL.Path, "file-read"):
						available := 5120
						if phase.Load() == 1 {
							available = 4096
						}
						json.NewEncoder(w).Encode(map[string]any{"data": map[string]any{"content": fmt.Sprintf("MemTotal: 8192 kB\nMemAvailable: %d kB\n", available)}})
					case strings.HasSuffix(r.URL.Path, "get-fsinfo"):
						fmt.Fprintf(w, `{"data":{"result":[{"mountpoint":"/","type":"ext4","total-bytes":%d,"used-bytes":%d}]}}`, 1000*mib, 300*mib)
					case strings.HasSuffix(r.URL.Path, "network-get-interfaces"):
						fmt.Fprint(w, `{"data":{"result":[{"name":"eth0","ip-addresses":[{"ip-address":"192.0.2.10","ip-address-type":"ipv4","prefix":24}]}]}}`)
					case strings.HasSuffix(r.URL.Path, "get-osinfo"):
						fmt.Fprint(w, `{"data":{"name":"Linux","version":"fixture"}}`)
					default:
						fmt.Fprint(w, `{"data":{"result":{"version":"1.0"}}}`)
					}
				default:
					http.NotFound(w, r)
				}
			}))
			defer server.Close()
			host := server.URL
			timeout := guestPollFixtureHangGuard
			if strings.HasPrefix(kind, "foreign ") {
				host += "/command-reply-fixture/" + strings.ReplaceAll(kind, " ", "-")
			}
			client, err := proxmox.NewClient(proxmox.ClientConfig{Host: host, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: timeout})
			if err != nil {
				t.Fatal(err)
			}
			m := &Monitor{config: &config.Config{}, rateTracker: NewRateTracker(), metricsHistory: NewMetricsHistory(32, time.Hour), guestMetadataLimiter: make(map[string]time.Time)}
			if preserveOrigins {
				m.state = models.NewState()
			}
			res := proxmox.ClusterResource{Type: "qemu", Node: "node", Name: "guest", VMID: 105, Status: "running", MaxMem: 8 * mib, Mem: 8 * mib, MaxDisk: 1000 * mib, CPU: 0.1}
			identity := "transport:node:105"
			var previous *models.VM
			build := func() (models.VM, string) {
				// The cycle starts before collection, as in pollVMsAndContainersEfficient.
				cycleStart := time.Now()
				vm, raw, source, notes, at, ok := m.buildVMFromClusterResource(context.Background(), "transport", res, client, identity, nil, previous)
				if !ok {
					t.Fatal("VM disappeared")
				}
				m.recordGuestSnapshot("transport", "qemu", "node", 105, GuestMemorySnapshot{Name: vm.Name, Status: vm.Status, RetrievedAt: at, MemorySource: source, Memory: vm.Memory, Raw: raw, Notes: notes})
				m.recordGuestMetrics([]models.VM{vm}, nil, cycleStart)
				previous = &vm
				if preserveOrigins {
					// The safe fixture's actual builder/broadcast output is also
					// available for consuming browser proof. Do not hand-author a
					// fresh/last-known carrier or replace its observation time.
					m.state.UpdateVMs([]models.VM{vm})
					front := m.buildBroadcastFrontendStateFromSnapshot(m.state.GetSnapshot(), m.mockModeFence.begin())
					if len(front.Resources) != 1 {
						t.Fatal("typed-reply producer lost the canonical guest")
					}
					wire, err := json.Marshal(map[string]any{
						"kind": kind, "phase": phase.Load(), "guestCommands": calls.Load(),
						"vm": vm, "resources": front.Resources,
						"history": m.metricsHistory.GetAllGuestMetrics(identity, time.Hour),
					})
					if err != nil {
						t.Fatal(err)
					}
					marker := "TYPED_REPLY_PRODUCER"
					if strings.HasPrefix(kind, "foreign ") {
						marker = "COMMAND_REPLY_PRODUCER"
					}
					t.Logf("%s %s", marker, wire)
				}
				return vm, source
			}
			initial, source := build()
			if source != "guest-agent-meminfo" || initial.Memory.Used != int64(3*mib) || initial.Disk.Used != int64(300*mib) || calls.Load() != 5 {
				t.Fatal("healthy baseline absent")
			}
			memKey, metadataKey := guestMemoryCacheKey("transport", "node", 105), guestMetadataCacheKey("transport", "node", 105)
			memory := m.vmAgentMemCache[memKey]
			metadata := m.guestMetadataCache[metadataKey]
			if !preserveOrigins {
				memory.fetchedAt = time.Now().Add(-2 * vmAgentMemCacheTTL)
				m.vmAgentMemCache[memKey] = memory
				metadata.fetchedAt = time.Now().Add(-2 * guestMetadataCacheTTL)
				m.guestMetadataCache[metadataKey] = metadata
			}
			// Typed-reply controls retain the real healthy read's original cache
			// and observation times. fetchedAt also owns measurement provenance;
			// manually aging it would invalidate an unchanged-origin assertion.
			// The filesystem is always read before these optional cached fields,
			// while the six-reader wire controls exercise fresh uncached clients.
			m.guestMetadataLimiter = make(map[string]time.Time)
			phase.Store(1)
			res.CPU = 0.2
			wantMemorySource := "previous-snapshot"
			if preserveOrigins {
				wantMemorySource = "guest-agent-meminfo" // The untouched warm cache retains its actual origin.
			}
			for i := 0; i < 2; i++ {
				deferred, source := build()
				wantReason := "prev-agent-cooldown"
				if i == 0 {
					// Filesystem now receives the original error, before optional
					// reads observe the resulting shared cooldown.
					wantReason = "prev-agent-completion-unverified"
					if kind == "lost reply" {
						wantReason = "prev-agent-timeout"
					} else if kind == "redirect" {
						wantReason = "prev-agent-redirect"
					}
				}
				if deferred.ID != initial.ID || deferred.GuestAgentStatus != "deferred" || deferred.DiskStatusReason != wantReason || deferred.Memory.Used != initial.Memory.Used || source != wantMemorySource || deferred.Disk.Used != initial.Disk.Used || !reflect.DeepEqual(deferred.NetworkInterfaces, initial.NetworkInterfaces) {
					t.Fatalf("uncertain command lost truthful continuity: source=%s vm=%+v", source, deferred)
				}
				if preserveOrigins && (deferred.Memory.Observation.State != "last-known" || !deferred.Memory.Observation.ObservedAt.Equal(initial.Memory.Observation.ObservedAt)) {
					t.Fatal("typed reply renewed or promoted the original memory observation")
				}
				if !reflect.DeepEqual(m.vmAgentMemCache[memKey], memory) || !reflect.DeepEqual(m.guestMetadataCache[metadataKey], metadata) {
					t.Fatal("uncertain command renewed or replaced cached evidence")
				}
			}
			if calls.Load() != 6 || redirected.Load() != 0 {
				t.Fatalf("commands/redirected after uncertainty = %d/%d, want 6/0", calls.Load(), redirected.Load())
			}
			for metric, want := range map[string]int{"cpu": 3, "memory": 1, "memoryused": 1, "disk": 1} {
				if got := len(m.metricsHistory.GetGuestMetrics(identity, metric, time.Hour)); got != want {
					t.Errorf("%s History points = %d, want %d", metric, got, want)
				}
			}
		})
	}
}