Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

10.5 KB
Raw Blame
package monitoring

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"net/http/httptest"
	"os"
	"reflect"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust"
	"github.com/rcourtman/pulse-go-rewrite/internal/recovery"
	"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
)

// Only storage catalog/content reads go through the real HTTP client. No guest
// commands, backup operation, external service or database-internal change.
type storageInventoryHTTPClient struct {
	*pveReconciliationClient
	api *proxmox.Client
}

func (c *storageInventoryHTTPClient) GetStorage(ctx context.Context, node string) ([]proxmox.Storage, error) {
	return c.api.GetStorage(ctx, node)
}

func (c *storageInventoryHTTPClient) GetStorageContent(ctx context.Context, node, storage string) ([]proxmox.StorageContent, error) {
	return c.api.GetStorageContent(ctx, node, storage)
}

type storageInventoryHTTPFixture struct {
	f      *pveReconciliationFixture
	client *storageInventoryHTTPClient
	mu     sync.Mutex
	bodies map[string]string
	paths  []string
}

func storageInventoryHTTPPath(kind, node string) string {
	path := "/api2/json/nodes/" + node + "/storage"
	if kind == "content" {
		path += "/archive-" + node + "/content"
	}
	return path
}

func newStorageInventoryHTTPFixture(t *testing.T) *storageInventoryHTTPFixture {
	t.Helper()
	h := &storageInventoryHTTPFixture{f: newPVEReconciliationFixture(t), bodies: make(map[string]string)}
	// Normal storage inventory supplies the carry-forward lookup when a node's
	// catalog cannot be read. Cold-state cases deliberately discard that cache.
	h.f.m.state.UpdateStorage([]models.Storage{
		{ID: "cluster-node-a-archive-node-a", Name: "archive-node-a", Node: "node-a", Instance: "cluster", Content: "backup", Type: "dir"},
		{ID: "cluster-node-b-archive-node-b", Name: "archive-node-b", Node: "node-b", Instance: "cluster", Content: "backup", Type: "dir"},
	})
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		h.mu.Lock()
		defer h.mu.Unlock()
		h.paths = append(h.paths, r.URL.Path)
		w.Header().Set("Content-Type", "application/json")
		if body, ok := h.bodies[r.URL.Path]; ok {
			fmt.Fprint(w, body)
			return
		}
		for _, node := range []string{"node-a", "node-b"} {
			var data any
			if r.URL.Path == storageInventoryHTTPPath("catalog", node) {
				data, _ = h.f.client.GetStorage(r.Context(), node)
			} else if r.URL.Path == storageInventoryHTTPPath("content", node) {
				data, _ = h.f.client.GetStorageContent(r.Context(), node, "archive-"+node)
			} else {
				continue
			}
			if err := json.NewEncoder(w).Encode(map[string]any{"data": data}); err != nil {
				t.Error(err)
			}
			return
		}
		t.Errorf("unexpected fixture request: %s", r.URL.Path)
		http.NotFound(w, r)
	}))
	t.Cleanup(server.Close)
	api, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: 5 * time.Second})
	if err != nil {
		t.Fatal(err)
	}
	h.client = &storageInventoryHTTPClient{pveReconciliationClient: h.f.client, api: api}
	return h
}

func (h *storageInventoryHTTPFixture) set(kind, node, body string) {
	h.mu.Lock()
	defer h.mu.Unlock()
	if body == "" {
		delete(h.bodies, storageInventoryHTTPPath(kind, node))
	} else {
		h.bodies[storageInventoryHTTPPath(kind, node)] = body
	}
}

func (h *storageInventoryHTTPFixture) advance() {
	h.mu.Lock()
	defer h.mu.Unlock()
	h.f.client.at = h.f.client.at.Add(15 * time.Minute)
}

func (h *storageInventoryHTTPFixture) poll(t *testing.T, wantReads int) {
	t.Helper()
	h.mu.Lock()
	before := len(h.paths)
	h.mu.Unlock()
	h.f.m.pollStorageBackupsWithNodes(context.Background(), "cluster", h.client, h.f.nodes, h.f.online)
	h.f.wait(t)
	h.mu.Lock()
	defer h.mu.Unlock()
	if len(h.paths)-before != wantReads {
		t.Fatalf("got HTTP reads %v, want %d; no retries or other operations", h.paths[before:], wantReads)
	}
}

func TestPVEStorageHTTPInvalidResponseRetainsHistory(t *testing.T) {
	for _, kind := range []string{"catalog", "content"} {
		row := `{"storage":"archive-node-b","type":"dir","content":"backup","enabled":1,"active":1}`
		missingDiscriminator := `{"storage":"archive-node-b","type":"dir"}`
		if kind == "content" {
			row = `{"volid":"archive-node-b:backup/vzdump-qemu-102.vma.zst","vmid":102,"content":"backup","ctime":1700000000}`
			missingDiscriminator = `{"volid":"archive-node-b:backup/vzdump-qemu-102.vma.zst"}`
		}
		cases := map[string]string{
			"missing-data":          `{}`,
			"null-data":             `{"data":null}`,
			"conflicting-error":     `{"data":[],"errors":{"message":"fixture-private-detail"}}`,
			"duplicate-data":        `{"data":[` + row + `],"data":[]}`,
			"trailing-value":        `{"data":[]} {"data":[]}`,
			"null-row":              `{"data":[null]}`,
			"bad-row-after-good":    `{"data":[` + row + `,null]}`,
			"missing-discriminator": `{"data":[` + missingDiscriminator + `]}`,
		}
		for _, cold := range []bool{false, true} {
			for name, body := range cases {
				t.Run(fmt.Sprintf("%s/cold=%t/%s", kind, cold, name), func(t *testing.T) {
					h := newStorageInventoryHTTPFixture(t)
					f := h.f
					h.poll(t, 4)
					other := recoveryIngestTestPoint("pve-backup:unrelated", "other-site", f.client.at)
					if err := f.store.UpsertPoints(context.Background(), []recovery.RecoveryPoint{other}); err != nil {
						t.Fatal(err)
					}
					before := f.points(t)
					if len(before) != 3 {
						t.Fatalf("healthy control points=%d, want 3", len(before))
					}
					beforeRows := make(map[string]models.StorageBackup)
					for _, row := range f.m.state.GetSnapshot().PVEBackups.StorageBackups {
						beforeRows[row.ID] = row
					}
					if cold {
						f.coldState()
					}
					h.advance()
					h.set(kind, "node-b", body)
					reads := 4
					if kind == "catalog" {
						reads = 3
					}
					for range 2 {
						h.poll(t, reads)
						after := f.points(t)
						if len(after) != 3 || !reflect.DeepEqual(before[other.ID], after[other.ID]) {
							t.Fatal("malformed success deleted history, admitted extra rows or changed another instance")
						}
						for id, point := range before {
							if point.SubjectResourceID == f.ids[1] && !reflect.DeepEqual(point, after[id]) {
								t.Fatal("unreadable backup was removed or its observation renewed")
							}
							if point.SubjectResourceID == f.ids[0] && (after[id].CompletedAt == nil || !after[id].CompletedAt.Equal(f.client.at)) {
								t.Fatal("independent accepted storage did not update")
							}
						}
						f.assertHistory(t, recovery.ProtectionHistoryPartial, operationaltrust.EvidencePermissionsUnknown)
						rows := f.m.state.GetSnapshot().PVEBackups.StorageBackups
						want := 2
						if cold {
							want = 1
						}
						if len(rows) != want {
							t.Fatalf("live rows=%d want=%d; unread history cannot erase warm rows or invent cold ones", len(rows), want)
						}
						if !cold {
							for _, row := range rows {
								if row.VMID == 102 && !reflect.DeepEqual(row, beforeRows[row.ID]) {
									t.Fatal("unread HTTP inventory changed its warm row")
								}
							}
						}
					}
					h.set(kind, "node-b", "")
					h.poll(t, 4)
					f.assertHistory(t, recovery.ProtectionHistoryComplete, operationaltrust.EvidencePermissionsSufficient)
					if len(f.m.state.GetSnapshot().PVEBackups.StorageBackups) != 2 {
						t.Fatal("accepted recovery did not restore live inventory")
					}
					// Explicit scoped absence still clears only this connection.
					h.set(kind, "node-a", `{"data":[]}`)
					h.set(kind, "node-b", `{"data":[]}`)
					reads = 4
					if kind == "catalog" {
						reads = 2
					}
					h.poll(t, reads)
					if points := f.points(t); len(points) != 1 || !reflect.DeepEqual(before[other.ID], points[other.ID]) || len(f.m.state.GetSnapshot().PVEBackups.StorageBackups) != 0 {
						t.Fatal("explicit empty response failed to clear exactly its own backup scope")
					}
				})
			}
		}
	}
}

func TestPVEStorageHTTPAllUnreadCannotClaimEmpty(t *testing.T) {
	for _, kind := range []string{"catalog", "content"} {
		for _, cold := range []bool{false, true} {
			t.Run(fmt.Sprintf("%s/cold=%t", kind, cold), func(t *testing.T) {
				h := newStorageInventoryHTTPFixture(t)
				h.poll(t, 4)
				before := h.f.points(t)
				if cold {
					h.f.coldState()
				}
				for _, node := range []string{"node-a", "node-b"} {
					h.set(kind, node, `{"data":null}`)
				}
				reads := 4
				if kind == "catalog" {
					reads = 2
				}
				for range 2 {
					h.poll(t, reads)
					if !reflect.DeepEqual(before, h.f.points(t)) {
						t.Fatal("all-unread response removed or renewed recovery evidence")
					}
					h.f.assertHistory(t, recovery.ProtectionHistoryUnavailable, operationaltrust.EvidencePermissionsUnknown)
				}
			})
		}
	}
}

// The browser consumes these actual HTTP -> poll -> queue -> store -> posture
// frames, not a separately invented provider state. Export stays opt-in.
func TestPVEStorageHTTPBrowserProducer(t *testing.T) {
	if os.Getenv("PULSE_STORAGE_HTTP_FRAMES") != "1" {
		t.Skip("producer export is opt-in")
	}
	h := newStorageInventoryHTTPFixture(t)
	frames := make(map[string]any)
	for _, phase := range []string{"observed", "catalog-unread", "content-unread", "cold-unread", "recovered", "confirmed-empty"} {
		switch phase {
		case "catalog-unread":
			h.set("catalog", "node-b", `{}`)
			h.advance()
		case "content-unread":
			h.set("catalog", "node-b", "")
			h.set("content", "node-b", `{"data":[{"volid":"unaccepted","content":"backup"},null]}`)
		case "cold-unread":
			h.f.coldState()
		case "recovered":
			h.set("content", "node-b", "")
		case "confirmed-empty":
			h.set("content", "node-a", `{"data":[]}`)
			h.set("content", "node-b", `{"data":[]}`)
		}
		reads := 4
		if phase == "catalog-unread" {
			reads = 3
		}
		h.poll(t, reads)
		points := h.f.points(t)
		want := 2
		if phase == "confirmed-empty" {
			want = 0
		}
		if len(points) != want {
			t.Fatalf("phase %s points=%d want=%d", phase, len(points), want)
		}
		frames[phase] = map[string]any{"pve": map[string]any{"data": h.f.m.state.GetSnapshot().PVEBackups},
			"postures": map[string]any{"data": h.f.postures(t), "policy": recovery.DefaultProtectionPosturePolicy.Payload()}, "points": points}
	}
	data, err := json.Marshal(map[string]any{"observed_at": time.Now().UTC(), "resource_ids": h.f.ids, "frames": frames, "paths": h.paths})
	if err != nil {
		t.Fatal(err)
	}
	// One Go-JSON event may hold only part of this line; collectors must join
	// all Output events before extracting the producer frame.
	t.Log("STORAGE_HTTP_FRAMES=" + strings.TrimSpace(string(data)))
}