Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

6.7 KB
Raw Blame
package monitoring

import (
	"context"
	"fmt"
	"reflect"
	"sync/atomic"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
)

func replicationOwner(t *testing.T, m *Monitor) *pveReplicationPoll {
	t.Helper()
	m.mu.RLock()
	owner := m.pveReplicationPolls["site-a"]
	m.mu.RUnlock()
	if owner == nil {
		t.Fatal("replication claim missing")
	}
	return owner
}
func TestReplicationLifecycleSingleFlightAndCycleCancellation(t *testing.T) {
	started := make(chan struct{})
	release := make(chan struct{})
	var calls atomic.Int32
	c := &lifecycleReplicationClient{read: func(ctx context.Context) ([]proxmox.ReplicationJob, error) {
		if calls.Add(1) == 1 {
			close(started)
		}
		select {
		case <-release:
			return []proxmox.ReplicationJob{{ID: "100-0", LastSyncStatus: "ok"}}, nil
		case <-ctx.Done():
			return nil, ctx.Err()
		}
	}}
	m := newReplicationLifecycleMonitor(t, c, true)
	runCtx, cancel := context.WithCancel(context.Background())
	defer cancel()
	m.setRuntimeContext(runCtx, nil)
	t.Cleanup(func() { cancel(); close(release) })
	runReplicationCycle(m)
	waitReplication(t, started, "first read")
	owner := replicationOwner(t, m)
	// executeScheduledTask canceled its short-lived task context on return.
	if owner.ctx.Err() != nil {
		t.Fatalf("cycle canceled background read: %v", owner.ctx.Err())
	}
	deadline, ok := owner.ctx.Deadline()
	if !ok || time.Until(deadline) > pveReplicationPollTimeout {
		t.Fatal("background read has no bounded deadline")
	}
	for i := 0; i < 5; i++ {
		runReplicationCycle(m)
	}
	if calls.Load() != 1 {
		t.Fatalf("overlapping cycles stacked %d reads", calls.Load())
	}
	cancel()
	waitReplication(t, owner.done, "runtime cancellation")
	if len(m.state.GetSnapshot().ReplicationJobs) != 0 {
		t.Fatal("cancellation published an inventory")
	}
	// A new ordinary runtime/cycle recovers without a manual inventory invocation.
	m.setRuntimeContext(context.Background(), nil)
	c.read = func(context.Context) ([]proxmox.ReplicationJob, error) {
		return []proxmox.ReplicationJob{{ID: "100-0", LastSyncStatus: "ok"}}, nil
	}
	runReplicationCycle(m)
	end := time.Now().Add(3 * time.Second)
	for len(m.state.GetSnapshot().ReplicationJobs) == 0 && time.Now().Before(end) {
		time.Sleep(time.Millisecond)
	}
	if len(m.state.GetSnapshot().ReplicationJobs) != 1 {
		t.Fatal("ordinary recovery did not publish")
	}
}
func TestReplicationLifecycleReplacementRejectsLateCompletion(t *testing.T) {
	for _, mode := range []string{"runtime", "client", "retire", "client-empty", "client-error"} {
		t.Run(mode, func(t *testing.T) {
			started := make(chan struct{})
			release := make(chan struct{})
			var released bool
			old := &lifecycleReplicationClient{read: func(ctx context.Context) ([]proxmox.ReplicationJob, error) {
				close(started)
				<-release
				return []proxmox.ReplicationJob{{ID: "stale-0", LastSyncStatus: "ok"}}, nil
			}}
			m := newReplicationLifecycleMonitor(t, old, true)
			m.setRuntimeContext(context.Background(), nil)
			before := replicationSeed(m)
			defer func() {
				if !released {
					close(release)
				}
			}()
			runReplicationCycle(m)
			waitReplication(t, started, "old read")
			owner := replicationOwner(t, m)
			switch mode {
			case "runtime":
				m.setRuntimeContext(context.Background(), nil) // Old parent deliberately still live.
			case "retire":
				m.retirePVEInstanceRuntime("site-a")
			}
			newer := &lifecycleReplicationClient{read: func(context.Context) ([]proxmox.ReplicationJob, error) {
				if mode == "client-empty" {
					return []proxmox.ReplicationJob{}, nil
				}
				if mode == "client-error" {
					return nil, fmt.Errorf("403 permission denied")
				}
				return []proxmox.ReplicationJob{{ID: "fresh-0", LastSyncStatus: "error"}}, nil
			}}
			m.mu.Lock()
			m.pveClients["site-a"] = newer
			m.lastClusterCheck["site-a"] = time.Now()
			m.mu.Unlock()
			// Dispatch through current provider binding, then wait for the owned terminal
			// boundary even when a successful empty result cannot be observed as a row.
			runReplicationCycle(m)
			// It may already have completed, so capture the terminal through state/claim.
			end := time.Now().Add(3 * time.Second)
			for {
				m.mu.RLock()
				current := m.pveReplicationPolls["site-a"]
				m.mu.RUnlock()
				if current == nil {
					break
				}
				if time.Now().After(end) {
					t.Fatal("replacement read did not terminate")
				}
				time.Sleep(time.Millisecond)
			}
			expected := m.state.GetSnapshot().ReplicationJobs
			if mode == "client-error" && !reflect.DeepEqual(expected, before) {
				t.Fatalf("failed replacement renewed inventory: %+v", expected)
			}
			if mode != "client-error" {
				expectedCount := 2
				if mode == "client-empty" {
					expectedCount = 1
				}
				if len(expected) != expectedCount {
					t.Fatalf("replacement result %+v", expected)
				}
				for _, row := range expected {
					if row.Instance == "site-a" && row.JobID != "fresh-0" {
						t.Fatalf("wrong replacement identity: %+v", row)
					}
				}
			}
			close(release)
			released = true
			waitReplication(t, owner.done, "old late completion")
			if got := m.state.GetSnapshot().ReplicationJobs; !reflect.DeepEqual(got, expected) {
				t.Fatalf("old read overwrote replacement: before=%+v after=%+v", expected, got)
			}
			if owner.ctx.Err() == nil {
				t.Fatal("superseded read not canceled")
			}
		})
	}
}

func TestReplicationLifecycleDeadlineAndOrdinaryRecovery(t *testing.T) {
	started := make(chan struct{})
	c := &lifecycleReplicationClient{read: func(ctx context.Context) ([]proxmox.ReplicationJob, error) {
		close(started)
		<-ctx.Done()
		return nil, ctx.Err()
	}}
	m := newReplicationLifecycleMonitor(t, c, true)
	m.setRuntimeContext(context.Background(), nil)
	before := replicationSeed(m)
	runReplicationCycle(m)
	waitReplication(t, started, "deadline-controlled read")
	owner := replicationOwner(t, m)
	select {
	case <-owner.done:
	case <-time.After(pveReplicationPollTimeout + 3*time.Second):
		t.Fatal("bounded read failed to terminate")
	}
	if owner.ctx.Err() != context.DeadlineExceeded {
		t.Fatalf("unexpected terminal context: %v", owner.ctx.Err())
	}
	if got := m.state.GetSnapshot().ReplicationJobs; !reflect.DeepEqual(got, before) {
		t.Fatalf("timed-out read renewed inventory: %+v", got)
	}
	c.read = func(context.Context) ([]proxmox.ReplicationJob, error) {
		return []proxmox.ReplicationJob{{ID: "100-0", GuestID: 100, LastSyncStatus: "ok"}}, nil
	}
	runReplicationCycle(m)
	end := time.Now().Add(3 * time.Second)
	for {
		rows := m.state.GetSnapshot().ReplicationJobs
		if len(rows) == 2 && rows[0].Instance == "site-a" && rows[0].LastSyncStatus == "ok" && rows[0].LastPolled.After(time.Unix(100, 0)) {
			break
		}
		if time.Now().After(end) {
			t.Fatalf("ordinary cycle did not recover after deadline: %+v", rows)
		}
		time.Sleep(time.Millisecond)
	}
}