Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

12.0 KB
Raw Blame
package monitoring

import (
	"context"
	"fmt"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/config"
	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/internal/recovery"
	recoverymanager "github.com/rcourtman/pulse-go-rewrite/internal/recovery/manager"
	proxmoxmapper "github.com/rcourtman/pulse-go-rewrite/internal/recovery/mapper/proxmox"
	"github.com/rcourtman/pulse-go-rewrite/internal/recovery/model"
	"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
	"github.com/stretchr/testify/require"
)

type cachedVzdumpClient struct {
	jobBackupTaskClient
	logErr error
}

func (c *cachedVzdumpClient) GetTaskLog(context.Context, string, string) ([]proxmox.TaskLogLine, error) {
	c.logRequests++
	return c.logLines, c.logErr
}

func TestVzdumpCachedGuestReadFailureDoesNotRenewRunningIntent(t *testing.T) {
	for _, failure := range []string{"503 unavailable", "403 denied"} {
		for _, parentStatus := range []string{"OK", "job errors"} {
			t.Run(failure+"/"+parentStatus, func(t *testing.T) {
				ctx := context.Background()
				job, lines := vzdumpGuestResultFixture()
				client := &cachedVzdumpClient{jobBackupTaskClient: jobBackupTaskClient{
					task:     proxmox.Task{UPID: testJobUPID, Node: job.Node, Type: "vzdump", StartTime: job.StartTime.Unix()},
					logLines: lines,
				}}
				manager := recoverymanager.New(config.NewMultiTenantPersistence(t.TempDir()))
				m := &Monitor{state: models.NewState(), recoveryManager: manager}
				store, err := manager.StoreForOrg("default")
				require.NoError(t, err)
				defer store.Close()
				other := models.BackupTask{ID: "other-job", Instance: "other", Node: job.Node, VMID: 103, Status: "running", ObservedAt: time.Now()}
				m.state.UpdateBackupTasksForInstance("other", []models.BackupTask{other})

				guests := func() []models.BackupTask {
					var result []models.BackupTask
					for _, task := range m.state.GetBackupTasks() {
						if task.Instance == "site" && task.VMID > 0 {
							result = append(result, task)
						}
						if task.Instance == "other" {
							require.Equal(t, other, task, "another connection must remain untouched")
						}
					}
					return result
				}
				// History intentionally ignores collections at or before its
				// millisecond watermark. These are distinct modeled poll cycles,
				// not duplicate observations within one persisted clock tick.
				poll := func() {
					t.Helper()
					coverage, err := store.BackupHistoryCoverage(ctx, job.StartTime, time.Now().UTC())
					require.NoError(t, err)
					for _, source := range coverage.Sources {
						if source.Provider == recovery.ProviderProxmoxPVE && source.Instance == "site" {
							if wait := time.Until(source.LastObservedAt.Add(time.Millisecond)); wait > 0 {
								time.Sleep(wait)
							}
							require.Greater(t, time.Now().UnixMilli(), source.LastObservedAt.UnixMilli())
						}
					}
					m.pollBackupTasks(ctx, "site", client)
				}
				checkStored := func(last recovery.Outcome, complete bool) {
					points, total, err := store.ListPoints(ctx, recovery.ListPointsOptions{Kind: recovery.KindOther})
					require.NoError(t, err)
					require.Equal(t, 4, total, "parent and three guest identities survive every poll")
					for _, point := range points {
						switch point.ID {
						case "pve-task:" + job.ID:
							require.Equal(t, complete, point.Details["guestTaskDetailsComplete"])
						case "pve-task:" + job.ID + "-vm101":
							require.Equal(t, recovery.OutcomeSuccess, point.Outcome)
						case "pve-task:" + job.ID + "-vm102":
							require.Equal(t, recovery.OutcomeFailed, point.Outcome)
						case "pve-task:" + job.ID + "-vm103":
							require.Equal(t, last, point.Outcome)
						default:
							t.Fatalf("unexpected point: %s", point.ID)
						}
					}
				}
				poll()
				assertVzdumpGuestResults(t, guests(), "running")
				checkStored(recovery.OutcomeRunning, false)
				_, active := m.resolveBackupIntentContext("", "site", job.Node, 103, time.Now())
				require.True(t, active, "an accepted live log still supplies backup intent")

				// Simulate passage of the evidence lease without a timing-dependent sleep.
				key := "site|" + testJobUPID
				entry := m.vzdumpJobTaskCache[key]
				observed := time.Now().UTC().Add(-2 * backupIntentEvidenceMaxAge)
				for i := range entry.tasks {
					require.False(t, entry.tasks[i].ObservedAt.IsZero(), "cache must retain its accepted log observation")
					entry.tasks[i].ObservedAt = observed
				}
				m.vzdumpJobTaskCache[key] = entry
				client.logErr = fmt.Errorf("task log %s", failure)
				poll()
				assertVzdumpGuestResults(t, guests(), "running")
				for _, task := range guests() {
					require.Equal(t, observed, task.ObservedAt, "failed reads cannot freshen the last log observation")
				}
				_, active = m.resolveBackupIntentContext("", "site", job.Node, 103, time.Now())
				require.False(t, active, "unread live logs must not indefinitely renew alert deferral")

				client.task.EndTime = job.EndTime.Unix()
				client.task.Status = parentStatus
				poll()
				assertVzdumpGuestResults(t, guests(), "unknown")
				for _, task := range guests() {
					require.Equal(t, observed, task.ObservedAt)
					require.False(t, task.EndTime.IsZero(), "ended parent must not leave an apparently active guest")
					if task.VMID == 103 {
						require.True(t, task.EndTime.Equal(job.EndTime), "compare the provider instant, not time.Location identity")
						require.Equal(t, "backup job ended without an observed guest result", task.Error)
					}
					_, active = m.resolveBackupIntentContext("", "site", job.Node, task.VMID, observed)
					require.False(t, active, "parent termination withdraws intent even while the cached observation is fresh")
				}
				checkStored(recovery.OutcomeUnknown, false)
				cached := m.vzdumpJobTaskCache[key]
				require.True(t, cached.lastAttempt.After(entry.lastAttempt), "failed reads still advance scheduling")
				entry.lastAttempt = cached.lastAttempt // scheduling is not guest observation evidence
				require.Equal(t, entry, cached, "fallback must not mutate/finalise accepted log data")
				require.Equal(t, 3, client.logRequests)

				// Only an ordinary accepted log supplies the missing guest verdict.
				client.logErr = nil
				client.logLines = append(lines, proxmox.TaskLogLine{LineNumber: 6, Text: "INFO: Finished Backup of VM 103 (00:02:00)"}, proxmox.TaskLogLine{LineNumber: 7, Text: "TASK ERROR: job errors"})
				poll()
				assertVzdumpGuestResults(t, guests(), "OK")
				checkStored(recovery.OutcomeSuccess, true)
				require.True(t, m.vzdumpJobTaskCache[key].final)
				for _, task := range guests() {
					require.True(t, task.ObservedAt.After(observed))
				}
				poll()
				require.Equal(t, 4, client.logRequests, "complete immutable logs still use the final cache")
				checkStored(recovery.OutcomeSuccess, true)
			})
		}
	}
}

func TestVzdumpCachedGuestUnreadLogUsesCurrentParent(t *testing.T) {
	for _, blocked := range []string{"budget", "cancelled"} {
		t.Run(blocked, func(t *testing.T) {
			job, lines := vzdumpGuestResultFixture()
			running := job
			running.EndTime = time.Time{}
			observed := time.Now().UTC().Add(-time.Minute)
			entry := vzdumpJobTaskCacheEntry{tasks: refreshObservedAt(parseVzdumpJobLog(running, testJobUPID, lines), observed)}
			m := &Monitor{vzdumpJobTaskCache: map[string]vzdumpJobTaskCacheEntry{"site|" + testJobUPID: entry}}
			client := &cachedVzdumpClient{}
			ctx := context.Background()
			jobs := []models.BackupTask{}
			upids := map[string]string{job.ID: testJobUPID}
			if blocked == "budget" {
				for i := 0; i < maxVzdumpJobLogFetchesPerPoll; i++ {
					// Compete within the ended-job queue. Running jobs now have
					// their own reserved turns, so they cannot fill this fixture's
					// entire allowance ahead of the ended cached parent.
					filler := models.BackupTask{ID: fmt.Sprintf("filler-%d", i), EndTime: job.EndTime}
					jobs = append(jobs, filler)
					upids[filler.ID] = filler.ID
				}
			} else {
				cancelled, cancel := context.WithCancel(ctx)
				cancel()
				ctx = cancelled
			}
			jobs = append(jobs, job)
			tasks := m.synthesizeVzdumpJobGuestTasks(ctx, "site", client, jobs, upids)
			assertVzdumpGuestResults(t, tasks, "unknown")
			for _, task := range tasks {
				require.Equal(t, observed, task.ObservedAt, "an unread log is not a fresh guest observation")
				require.False(t, task.EndTime.IsZero())
			}
			require.Equal(t, entry, m.vzdumpJobTaskCache["site|"+testJobUPID])
			wantRequests := 0
			if blocked == "budget" {
				wantRequests = maxVzdumpJobLogFetchesPerPoll
			}
			require.Equal(t, wantRequests, client.logRequests, "repair must not bypass request or cancellation bounds")
		})
	}
}

// Pin the distinction behind the cached-guest watchdog failure without relying
// on machine speed, sleeps, repeated test executions or a changed store clock.
// Live state can accept a parent/log transition before the collection timestamp
// advances far enough to be a distinct durable observation.
func TestVzdumpCachedGuestHistoryCollectionPrecision(t *testing.T) {
	ctx := context.Background()
	job, lines := vzdumpGuestResultFixture()
	ended := job
	job.Status, job.EndTime = "running", time.Time{}
	client := &cachedVzdumpClient{jobBackupTaskClient: jobBackupTaskClient{logLines: lines}}
	manager := recoverymanager.New(config.NewMultiTenantPersistence(t.TempDir()))
	m := &Monitor{recoveryManager: manager}
	store, err := manager.StoreForOrg("default")
	require.NoError(t, err)
	defer store.Close()
	at := time.Now().UTC().Add(-time.Minute).Truncate(time.Millisecond)
	collection := model.BackupHistoryCollection{
		Provider: recovery.ProviderProxmoxPVE, Instance: "site",
		ObservedAt: at, Complete: true, MaxGap: time.Minute,
	}
	upids := map[string]string{job.ID: testJobUPID}
	capture := func(phase, liveStatus string, storedOutcome recovery.Outcome, complete bool) {
		t.Helper()
		guests := m.synthesizeVzdumpJobGuestTasks(ctx, "site", client, []models.BackupTask{job}, upids)
		assertVzdumpGuestResults(t, guests, liveStatus)
		points, err := proxmoxmapper.FromPVEBackupTasksWithEvidence(append([]models.BackupTask{job}, guests...), nil, collection.ObservedAt)
		require.NoError(t, err)
		m.qualifyBackupGuestTaskDetails("site", []models.BackupTask{job}, upids, points)
		m.persistBackupTaskHistory(ctx, points, collection)
		stored, total, err := store.ListPoints(ctx, recovery.ListPointsOptions{Kind: recovery.KindOther})
		require.NoError(t, err)
		require.Equal(t, 4, total)
		for _, point := range stored {
			switch point.ID {
			case "pve-task:" + job.ID:
				require.Equal(t, complete, point.Details["guestTaskDetailsComplete"], phase)
			case "pve-task:" + job.ID + "-vm101":
				require.Equal(t, recovery.OutcomeSuccess, point.Outcome, phase)
			case "pve-task:" + job.ID + "-vm102":
				require.Equal(t, recovery.OutcomeFailed, point.Outcome, phase)
			case "pve-task:" + job.ID + "-vm103":
				require.Equal(t, storedOutcome, point.Outcome, phase)
			default:
				t.Fatalf("%s: unexpected point %q", phase, point.ID)
			}
		}
		t.Logf("%s collection=%s live=%s durable=%s complete=%t", phase, collection.ObservedAt.Format(time.RFC3339Nano), liveStatus, storedOutcome, complete)
	}
	capture("running", "running", recovery.OutcomeRunning, false)
	job = ended
	client.logErr = fmt.Errorf("task log 503 unavailable")
	collection.ObservedAt = at.Add(time.Millisecond - time.Nanosecond)
	capture("ended-same-tick", "unknown", recovery.OutcomeRunning, false)
	collection.ObservedAt = at.Add(time.Millisecond)
	capture("ended-next-tick", "unknown", recovery.OutcomeUnknown, false)
	client.logErr = nil
	client.logLines = append(lines, proxmox.TaskLogLine{LineNumber: 6, Text: "INFO: Finished Backup of VM 103 (00:02:00)"}, proxmox.TaskLogLine{LineNumber: 7, Text: "TASK ERROR: job errors"})
	collection.ObservedAt = at.Add(2*time.Millisecond - time.Nanosecond)
	capture("complete-same-tick", "OK", recovery.OutcomeUnknown, false)
	collection.ObservedAt = at.Add(2 * time.Millisecond)
	capture("complete-next-tick", "OK", recovery.OutcomeSuccess, true)
	require.True(t, m.vzdumpJobTaskCache["site|"+testJobUPID].final)
	require.Equal(t, 4, client.logRequests, "the complete log remains immutable after recovery")
}