Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

5.6 KB
Raw Blame
package monitoring

import (
	"context"
	"fmt"
	"reflect"
	"strings"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/recovery"
	"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
)

// One enabled but unreachable PBS storage on one node, beside ordinary
// readable archive storage on every node: the shape that turned every guest on
// a production connection to attention.
type pveDeadOffsiteStorageClient struct {
	*pveReconciliationClient
	offsiteErr error
}

const deadOffsiteStorage = "pbs-energy-offsite"

func (c *pveDeadOffsiteStorageClient) GetStorage(ctx context.Context, node string) ([]proxmox.Storage, error) {
	rows, err := c.pveReconciliationClient.GetStorage(ctx, node)
	if err == nil && node == "node-a" {
		rows = append(rows, proxmox.Storage{Storage: deadOffsiteStorage, Type: "pbs", Content: "backup", Shared: 1, Active: 1, Enabled: 1})
	}
	return rows, err
}

func (c *pveDeadOffsiteStorageClient) GetStorageContent(ctx context.Context, node, storage string) ([]proxmox.StorageContent, error) {
	if storage == deadOffsiteStorage {
		if c.offsiteErr != nil {
			return nil, c.offsiteErr
		}
		return nil, nil
	}
	return c.pveReconciliationClient.GetStorageContent(ctx, node, storage)
}

func TestPVEUnreadStorageLeavesReadBackupsProtected(t *testing.T) {
	f := newPVEReconciliationFixture(t)
	c := &pveDeadOffsiteStorageClient{
		pveReconciliationClient: f.client,
		offsiteErr:              fmt.Errorf("pbs-energy-offsite: error fetching datastores - 127.0.0.1:18007: connection refused"),
	}
	poll := func() {
		f.m.pollStorageBackupsWithNodes(context.Background(), "cluster", c, f.nodes, f.online)
		f.wait(t)
	}
	wantUnread := []recovery.ProtectionUnreadScope{{
		Kind: recovery.ProtectionUnreadScopeStorage, Node: "node-a", Storage: deadOffsiteStorage,
	}}
	assertNamed := func(p recovery.ProtectionPosture) {
		t.Helper()
		if len(p.ProviderStates) != 1 ||
			p.ProviderStates[0].HistoryCompleteness != recovery.ProtectionHistoryPartial ||
			!reflect.DeepEqual(p.ProviderStates[0].UnreadScopes, wantUnread) {
			t.Errorf("provider state lost the unread storage: %+v", p.ProviderStates)
		}
		if !strings.Contains(p.Explanation, "could not read "+deadOffsiteStorage+" on node-a") {
			t.Errorf("explanation does not name the unread storage: %q", p.Explanation)
		}
	}

	// Fresh backups on storages that read fine are protected; the dead
	// storage limits coverage and is named, but proves nothing about them.
	poll()
	for _, p := range f.postures(t) {
		if p.State != recovery.ProtectionStateProtected || p.Coverage != recovery.ProtectionCoveragePartial ||
			p.Freshness != recovery.ProtectionFreshnessCurrent {
			t.Errorf("read backup was not protected beside one unread storage: %+v", p)
		}
		assertNamed(p)
	}

	// Node-b's archive is pruned while the offsite storage is still dead. The
	// partial poll cannot delete the stored point, but it did not re-read it,
	// so that retained point no longer supports protection.
	retained := f.points(t)
	f.client.at = f.client.at.Add(15 * time.Minute)
	f.client.emptyB = true
	poll()
	points := f.points(t)
	if len(points) != len(retained) {
		t.Fatalf("partial poll deleted history: points=%d want %d", len(points), len(retained))
	}
	for _, p := range f.postures(t) {
		switch p.SubjectResourceID {
		case f.ids[0]:
			if p.State != recovery.ProtectionStateProtected {
				t.Errorf("re-read backup lost protection: %+v", p)
			}
		case f.ids[1]:
			if p.State == recovery.ProtectionStateProtected || p.Coverage != recovery.ProtectionCoveragePartial {
				t.Errorf("retained point that the latest poll did not read claimed protection: %+v", p)
			}
		}
		assertNamed(p)
	}

	// Once the storage reads again the collection is complete and reconciles
	// the pruned archive away; the unread names disappear with the gap.
	c.offsiteErr = nil
	poll()
	for _, p := range f.postures(t) {
		if len(p.ProviderStates) > 0 && len(p.ProviderStates[0].UnreadScopes) != 0 {
			t.Errorf("complete collection kept unread scopes: %+v", p.ProviderStates)
		}
		if p.SubjectResourceID == f.ids[0] &&
			(p.State != recovery.ProtectionStateProtected || p.Coverage != recovery.ProtectionCoverageComplete) {
			t.Errorf("complete read did not restore full protection: %+v", p)
		}
		if p.SubjectResourceID == f.ids[1] && p.State == recovery.ProtectionStateProtected {
			t.Errorf("confirmed-absent archive still protected its guest: %+v", p)
		}
	}
}

func TestPVEUnreadProtectionScopesNameWhatWentUnread(t *testing.T) {
	got := pveUnreadProtectionScopes(pveUnreadReads{
		nodes: map[string]struct{}{"node-c": {}},
		local: map[pveLocalBackupStorage]struct{}{{node: "node-b", storage: "local"}: {}},
		shared: map[string]struct{}{
			"offsite": {},
			"nfs":     {},
		},
		sharedReads: map[pveLocalBackupStorage]struct{}{
			{node: "node-a", storage: "offsite"}: {},
			{node: "node-a", storage: "nfs"}:     {},
			{node: "node-b", storage: "nfs"}:     {},
		},
		denied: map[pveLocalBackupStorage]struct{}{
			{node: "node-c"}:                     {},
			{node: "node-b", storage: "nfs"}:     {},
			{node: "node-a", storage: "unknown"}: {},
		},
	})
	want := []recovery.ProtectionUnreadScope{
		{Kind: recovery.ProtectionUnreadScopeNode, Node: "node-c", AccessDenied: true},
		{Kind: recovery.ProtectionUnreadScopeStorage, Node: "node-b", Storage: "local"},
		// Several nodes failed the same shared storage: name the storage only.
		{Kind: recovery.ProtectionUnreadScopeStorage, Storage: "nfs", AccessDenied: true},
		{Kind: recovery.ProtectionUnreadScopeStorage, Node: "node-a", Storage: "offsite"},
	}
	if !reflect.DeepEqual(got, want) {
		t.Fatalf("unread scopes = %+v\nwant %+v", got, want)
	}
}