Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

4.9 KB
Raw Blame
package monitoring

import (
	"encoding/json"
	"fmt"
	"reflect"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust"
	"github.com/rcourtman/pulse-go-rewrite/internal/recovery"
)

func conflictingPBSSnapshotBody(kind, failure string, at int64) string {
	row := map[string]any{"backup-type": kind, "backup-id": "101", "backup-time": at, "size": 1024, "files": []string{"index.json.blob"}, "verification": map[string]any{"state": "ok"}, "comment": "untrusted-new-file"}
	rows := []map[string]any{row}
	switch failure {
	case "wrong-kind":
		if kind == "vm" {
			row["backup-type"] = "ct"
		} else {
			row["backup-type"] = "vm"
		}
	case "wrong-id":
		row["backup-id"] = "100"
	case "missing-kind":
		delete(row, "backup-type")
	case "missing-id":
		delete(row, "backup-id")
	case "identical-repeat":
		rows = append(rows, row)
	case "conflicting-repeat":
		rows = append(rows, map[string]any{"backup-type": kind, "backup-id": "101", "backup-time": at, "size": 2048, "verification": map[string]any{"state": "failed"}})
	}
	b, _ := json.Marshal(map[string]any{"data": rows})
	return string(b)
}

func TestPBSConflictingSnapshotIdentityRetainsUnreadScope(t *testing.T) {
	for _, kind := range []string{"vm", "ct"} {
		for _, failure := range []string{"wrong-kind", "wrong-id", "missing-kind", "missing-id", "identical-repeat", "conflicting-repeat"} {
			for _, warm := range []bool{false, true} {
				t.Run(fmt.Sprintf("%s/%s/warm-%t", kind, failure, warm), func(t *testing.T) {
					f := newPBSNamespaceContinuityFixture(t, kind)
					if warm {
						f.collect(t, true, nil)
					}
					before, points := f.live(), f.points(t, "pbs-main")
					key := pbsBackupGroupKey{datastore: "archive", namespace: "tenant", backupType: kind, backupID: "101"}
					cacheAt := f.m.pbsBackupCacheTimeFor("pbs-main", key)
					f.rootAt.Add(15 * 60)
					f.tenantAt.Add(15 * 60)
					f.inventoryResponse.Store(&pbsInventoryResponse{stage: "snapshots", body: conflictingPBSSnapshotBody(kind, failure, f.tenantAt.Load())})
					for range 2 {
						f.collect(t, true, nil)
						raw := f.m.state.GetSnapshot().PBSBackups
						want := 1
						if warm {
							want = 2
						}
						if len(raw) != want || f.live()["100"].BackupTime.Unix() != f.rootAt.Load() {
							t.Errorf("invalid read displaced independent scope or invented files: %+v", raw)
						}
						for _, row := range raw {
							if row.Namespace != "tenant" {
								continue
							}
							if !warm || !reflect.DeepEqual(row, before["101"]) {
								t.Errorf("malformed group published or rewrote raw history: %+v", row)
							}
						}
						if !f.m.pbsBackupCacheTimeFor("pbs-main", key).Equal(cacheAt) {
							t.Error("invalid response renewed cache")
						}
						for _, prior := range points {
							if prior.Details["namespace"] != "tenant" {
								continue
							}
							found := false
							for _, now := range f.points(t, "pbs-main") {
								found = found || reflect.DeepEqual(prior, now)
							}
							if !found {
								t.Error("unread historical point was deleted or renewed")
							}
						}
						for _, p := range f.postures(t) {
							// The root sibling was read; the malformed tenant group was not.
							if (p.State == recovery.ProtectionStateProtected) != (p.SubjectResourceID == f.ids[0]) {
								t.Errorf("invalid group protection does not follow the read root: %+v", p)
							}
							for _, provider := range p.ProviderStates {
								if provider.HistoryCompleteness != recovery.ProtectionHistoryPartial || provider.Permissions != operationaltrust.EvidencePermissionsUnknown {
									t.Errorf("malformed read treated as complete or access refusal: %+v", provider)
								}
							}
						}
					}
					if f.failedReads.Load() != 2 {
						t.Errorf("malformed response caused extra reads or poisoned TTL reuse: %d", f.failedReads.Load())
					}
					f.inventoryResponse.Store(nil)
					f.collect(t, true, nil)
					if len(f.live()) != 2 || f.live()["101"].BackupTime.Unix() != f.tenantAt.Load() || !f.m.pbsBackupCacheTimeFor("pbs-main", key).After(cacheAt) {
						t.Error("ordinary valid read did not recover without cache reset")
					}
					for _, p := range f.postures(t) {
						if p.State != recovery.ProtectionStateProtected {
							t.Errorf("valid recovery not protected: %+v", p)
						}
					}
					// A valid empty snapshot array is still authoritative for this
					// group; the independent root namespace must survive it.
					f.tenantAt.Add(int64(time.Hour / time.Second))
					f.inventoryResponse.Store(&pbsInventoryResponse{stage: "snapshots", body: `{"data":[]}`})
					f.collect(t, true, nil)
					if len(f.live()) != 1 || f.live()["100"].VMID != "100" {
						t.Error("confirmed empty snapshots did not remove only their own group")
					}
					points = f.points(t, "pbs-main")
					if len(points) != 1 || points[0].Details["namespace"] != "" {
						t.Error("complete empty snapshot read did not reconcile only its own group")
					}
				})
			}
		}
	}
}