Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

6.8 KB
Raw Blame
package monitoring

import (
	"context"
	"encoding/json"
	"net/http"
	"net/http/httptest"
	"strings"
	"sync/atomic"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/internal/recovery"
	proxmoxmapper "github.com/rcourtman/pulse-go-rewrite/internal/recovery/mapper/proxmox"
	"github.com/rcourtman/pulse-go-rewrite/pkg/pbs"
)

// Exercise every abort boundary after a successful refresh, without flushing
// cache on recovery. A timestamp for an unpublished row must not hide a failed
// verification behind the previously published verified snapshot.
func TestPBSOrdinaryPollCancellationDoesNotRenewUnpublishedCache(t *testing.T) {
	for _, boundary := range []string{"before-datastore", "mid-datastore", "after-enumeration", "during-task-correlation"} {
		t.Run(boundary, func(t *testing.T) {
			f := newPBSPosturePollFixture(t, 1)
			var phase, snapshotReads atomic.Int32
			blocked := make(chan struct{}, 1)
			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				aborting := phase.Load() == 1
				isOther := strings.Contains(r.URL.Path, "/other/") || r.URL.Query().Get("ns") == "other"
				isTask := strings.HasSuffix(r.URL.Path, "/tasks")
				if aborting && ((boundary != "during-task-correlation" && isOther) || (boundary == "during-task-correlation" && isTask)) {
					blocked <- struct{}{}
					<-r.Context().Done()
					return
				}
				if strings.HasSuffix(r.URL.Path, "/groups") {
					groups := []map[string]any{}
					if strings.Contains(r.URL.Path, "/archive/") && r.URL.Query().Get("ns") == "" {
						groups = append(groups, map[string]any{"backup-type": "vm", "backup-id": "100", "last-backup": f.backupAt, "backup-count": 1})
					}
					_ = json.NewEncoder(w).Encode(map[string]any{"data": groups})
					return
				}
				if strings.HasSuffix(r.URL.Path, "/snapshots") {
					snapshotReads.Add(1)
					verification := "ok"
					if phase.Load() != 0 {
						verification = "failed"
					}
					snapshot := map[string]any{"backup-type": "vm", "backup-id": "100", "backup-time": f.backupAt, "size": 1024, "files": []string{"index.json.blob"}, "verification": map[string]any{"state": verification}}
					if aborting && boundary == "during-task-correlation" {
						// This forces the final asynchronous HTTP read after enumeration.
						snapshot["size"] = 0
						snapshot["files"] = []string{"qemu-server.conf.blob"}
					}
					_ = json.NewEncoder(w).Encode(map[string]any{"data": []map[string]any{snapshot}})
					return
				}
				http.NotFound(w, r)
			}))
			t.Cleanup(server.Close)
			var err error
			f.client, err = pbs.NewClient(pbs.ClientConfig{Host: server.URL, TokenName: "fixture@pbs!reader", TokenValue: "fixture-only"})
			if err != nil {
				t.Fatal(err)
			}
			datastores := []models.PBSDatastore{{Name: "archive"}, {Name: "other"}}
			switch boundary {
			case "before-datastore":
				datastores = append(datastores, models.PBSDatastore{Name: "never"})
			case "mid-datastore":
				datastores = []models.PBSDatastore{{Name: "archive", Namespaces: []models.PBSNamespace{{Path: ""}, {Path: "other"}, {Path: "never"}}}}
			case "during-task-correlation":
				datastores = []models.PBSDatastore{{Name: "archive"}}
			}
			f.m.pollPBSBackups(context.Background(), "pbs-main", f.client, datastores, nil)
			f.wait(t)
			f.assertProtected(t, "pbs-main")
			key := pbsBackupGroupKey{datastore: "archive", backupType: "vm", backupID: "100"}
			expired := time.Now().Add(-2 * pbsBackupCacheTTL)
			f.m.setPBSBackupCacheTime("pbs-main", key, expired)
			phase.Store(1)
			ctx, cancel := context.WithCancel(context.Background())
			defer cancel()
			done := make(chan struct{})
			go func() {
				defer close(done)
				f.m.pollPBSBackups(ctx, "pbs-main", f.client, datastores, nil)
			}()
			select {
			case <-blocked:
				cancel()
			case <-time.After(10 * time.Second):
				cancel()
				<-done
				t.Fatal("poll did not reach the intended cancellation boundary")
			}
			select {
			case <-done:
			case <-time.After(10 * time.Second):
				t.Fatal("cancelled poll did not stop")
			}
			f.wait(t)
			if got := f.m.pbsBackupCacheTimeFor("pbs-main", key); !got.Equal(expired) {
				t.Errorf("cancelled poll renewed unpublished cache: got %v, want %v", got, expired)
			}
			raw := f.m.state.GetSnapshot().PBSBackups
			if len(raw) != 1 || !raw[0].Verified || raw[0].InProgress {
				t.Errorf("cancelled poll replaced published raw rows: %+v", raw)
			}
			points := f.points(t, "pbs-main")
			if len(points) != 1 || points[0].Verified == nil || !*points[0].Verified {
				t.Errorf("cancelled poll replaced persisted evidence: %+v", points)
			}
			// Do not clear or expire cache again. The next ordinary poll must
			// independently fetch the verification change the aborted poll lost.
			phase.Store(2)
			f.m.pollPBSBackups(context.Background(), "pbs-main", f.client, datastores, nil)
			f.wait(t)
			raw = f.m.state.GetSnapshot().PBSBackups
			mapped, err := proxmoxmapper.FromPBSBackupsWithEvidence(raw, buildPBSGuestCandidates(f.m.GetUnifiedReadStateOrSnapshot()), time.Now())
			points = f.points(t, "pbs-main")
			postures := f.postures(t)
			if snapshotReads.Load() != 3 || len(raw) != 1 || raw[0].Verified || raw[0].InProgress || err != nil || len(mapped) != 1 || mapped[0].Verified == nil || *mapped[0].Verified || len(points) != 1 || points[0].Verified == nil || *points[0].Verified || postures[0].State != recovery.ProtectionStateAttention || postures[0].Verification != recovery.ProtectionVerificationUnverified {
				t.Errorf("recovery reads=%d raw=%+v mapped=%+v err=%v persisted=%+v posture=%+v", snapshotReads.Load(), raw, mapped, err, points, postures)
			}
			if len(mapped) != 1 || len(points) != 1 {
				t.Fatal("recovery did not retain exactly one mapped and persisted point")
			}
			if mapped[0].SubjectResourceID != f.ids[0] || points[0].SubjectResourceID != f.ids[0] {
				t.Errorf("recovery changed subject identity: mapped=%s persisted=%s want=%s", mapped[0].SubjectResourceID, points[0].SubjectResourceID, f.ids[0])
			}
			publishedAt := f.m.pbsBackupCacheTimeFor("pbs-main", key)
			if !publishedAt.After(expired) {
				t.Error("successful recovery did not renew its published cache")
			}
			// Once the matching failed row has been published, normal TTL reuse
			// must avoid an unnecessary fourth read without restoring verified.
			f.m.pollPBSBackups(context.Background(), "pbs-main", f.client, datastores, nil)
			f.wait(t)
			if snapshotReads.Load() != 3 || !f.m.pbsBackupCacheTimeFor("pbs-main", key).Equal(publishedAt) || f.postures(t)[0].Verification != recovery.ProtectionVerificationUnverified {
				t.Error("cache-hit poll reread or renewed snapshots, or restored stale verification")
			}
			t.Logf("boundary=%s snapshots=%d raw/mapped/persisted=%d/%d/%d subject=%s posture=%s/%s", boundary, snapshotReads.Load(), len(raw), len(mapped), len(points), f.ids[0], postures[0].State, postures[0].Verification)
		})
	}
}