Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

10.5 KB
Raw Blame
package api

import (
	"context"
	"encoding/json"
	"net/http"
	"net/http/httptest"
	"os"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/config"
	"github.com/rcourtman/pulse-go-rewrite/internal/recovery"
	recoverymanager "github.com/rcourtman/pulse-go-rewrite/internal/recovery/manager"
	"github.com/rcourtman/pulse-go-rewrite/internal/recovery/model"
	"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
	"github.com/rcourtman/pulse-go-rewrite/pkg/reporting"
	"github.com/stretchr/testify/require"
)

// Public tests exercise the scoped evidence contract and shared delivery.
// Real private renderer output is tested in pulse-enterprise and the browser.
func installBackupHealthTestRenderer(t *testing.T) {
	t.Helper()
	previous := resolveBackupHealthReportGenerator()
	SetBackupHealthReportGenerator(func(data reporting.BackupHealthData, _ reporting.ReportFormat) ([]byte, string, error) {
		bytes, err := json.Marshal(data)
		return bytes, "application/json", err
	})
	t.Cleanup(func() { SetBackupHealthReportGenerator(previous) })
}
func readBackupHealthTestPayload(t *testing.T, data []byte) reporting.BackupHealthData {
	t.Helper()
	var payload reporting.BackupHealthData
	require.NoError(t, json.Unmarshal(data, &payload))
	return payload
}

func TestBackupHealthRequiresPrivateRenderer(t *testing.T) {
	previous := resolveBackupHealthReportGenerator()
	SetBackupHealthReportGenerator(nil)
	t.Cleanup(func() { SetBackupHealthReportGenerator(previous) })
	h, _, ctx, s, _ := inventoryScheduleFixture(t)
	s.Kind = config.ReportScheduleKindBackupHealth
	_, err := h.generateBackupHealthReport(ctx, s, time.Now(), true)
	require.ErrorContains(t, err, "rendering is unavailable")
	w := httptest.NewRecorder()
	h.HandleGetReportingCatalog(w, httptest.NewRequest(http.MethodGet, "/", nil))
	var catalog reporting.ReportingCatalog
	require.NoError(t, json.Unmarshal(w.Body.Bytes(), &catalog))
	require.False(t, catalog.BackupHealthAvailable)
	require.NotContains(t, w.Body.String(), "backupHealthAvailable", "runtimes without the extension retain the existing catalog shape")
	installBackupHealthTestRenderer(t)
	w = httptest.NewRecorder()
	h.HandleGetReportingCatalog(w, httptest.NewRequest(http.MethodGet, "/", nil))
	require.NoError(t, json.Unmarshal(w.Body.Bytes(), &catalog))
	require.True(t, catalog.BackupHealthAvailable)
}

func TestBackupHealthWeeklyWindowUsesSevenCompletedLocalDaysAcrossDST(t *testing.T) {
	installBackupHealthTestRenderer(t)
	h, _, ctx, s, _ := inventoryScheduleFixture(t)
	s.Kind = config.ReportScheduleKindBackupHealth
	s.Cadence.Timezone = "Europe/London"
	now := time.Date(2026, 10, 27, 10, 0, 0, 0, time.UTC)
	report, err := h.generateBackupHealthReport(ctx, s, now, true)
	require.NoError(t, err)
	require.Equal(t, time.Date(2026, 10, 27, 0, 0, 0, 0, time.UTC), report.End)
	require.Equal(t, 169*time.Hour, report.End.Sub(report.Start))
	require.Len(t, readBackupHealthTestPayload(t, report.Data).Summary.Days, 7)
}

func TestBackupHealthScheduleDataQualityCannotBeForgedByClient(t *testing.T) {
	h, p, ctx, s, _ := inventoryScheduleFixture(t)
	s.Kind = config.ReportScheduleKindBackupHealth
	s.Enabled = false
	s.LastReportDataState = "complete"
	s.LastReportDataMessage = "untrusted completeness claim"
	w := httptest.NewRecorder()
	h.HandleCreateReportSchedule(w, httptest.NewRequest(http.MethodPost, "/api/admin/reports/schedules", encodeScheduleBody(t, s)).WithContext(ctx))
	require.Equal(t, http.StatusCreated, w.Code, w.Body.String())
	created := storedReport(t, p)
	require.Empty(t, created.LastReportDataState)
	require.Empty(t, created.LastReportDataMessage)
}

func TestBackupHealthScheduleReportsRetainedFailuresBeyondPerformanceCap(t *testing.T) {
	installBackupHealthTestRenderer(t)
	h, p, ctx, s, _ := inventoryScheduleFixture(t)
	h.recoveryManager = recoverymanager.New(config.NewMultiTenantPersistence(t.TempDir()))
	store, err := h.recoveryManager.StoreForOrg("default")
	require.NoError(t, err)
	defer store.Close()
	now := time.Now().UTC().Truncate(time.Second)
	at := now.Add(-48 * time.Hour)
	point := recovery.RecoveryPoint{ID: "pve-task:fixture-failed", Provider: recovery.ProviderProxmoxPVE, Kind: recovery.KindOther, Mode: recovery.ModeLocal, Outcome: recovery.OutcomeFailed, StartedAt: &at, CompletedAt: &at, Details: map[string]any{"instance": "fixture", "vmid": 100, "status": "ERROR: storage unavailable"}}
	observation := model.BackupHistoryCollection{Provider: recovery.ProviderProxmoxPVE, Instance: "fixture", ObservedAt: now.Add(-8 * 24 * time.Hour), Complete: true, MaxGap: 9 * 24 * time.Hour}
	require.NoError(t, store.RecordBackupHistory(ctx, []recovery.RecoveryPoint{point}, []model.BackupHistoryCollection{observation}))
	observation.ObservedAt = now
	require.NoError(t, store.RecordBackupHistory(ctx, nil, []model.BackupHistoryCollection{observation}))
	s.Kind = config.ReportScheduleKindBackupHealth
	s, err = h.prepareReportSchedule(ctx, s, now, false)
	require.NoError(t, err)
	generated, err := h.generateBackupHealthReport(ctx, s, now, true)
	require.NoError(t, err)
	require.Equal(t, 61, generated.ResourceCount)
	require.Equal(t, "complete", generated.DataState)
	require.Equal(t, 0, generated.End.UTC().Hour())
	require.Equal(t, generated.End.AddDate(0, 0, -7), generated.Start)
	payload := readBackupHealthTestPayload(t, generated.Data)
	require.Equal(t, 1, payload.Summary.Counts.Failed)
	require.Len(t, payload.Summary.Workloads, 61)
	require.Len(t, payload.Summary.Failures, 1)
	require.Contains(t, payload.Summary.Failures[0].Detail, "storage unavailable")
	result, finished := h.runReportSchedule(ctx, p, s, now, true, "")
	require.Equal(t, "ok", finished.LastRunStatus, finished.LastError)
	require.Equal(t, "complete", finished.LastReportDataState)
	require.NotEmpty(t, finished.LastReportDataMessage)
	data, err := os.ReadFile(result.path)
	require.NoError(t, err)
	require.Contains(t, string(data), "fixture-failed")
}

func TestBackupHealthUnavailableHistoryDoesNotLookLikeZeroFailures(t *testing.T) {
	installBackupHealthTestRenderer(t)
	h, _, ctx, s, _ := inventoryScheduleFixture(t)
	s.Kind = config.ReportScheduleKindBackupHealth
	report, err := h.generateBackupHealthReport(ctx, s, time.Now().UTC(), true)
	require.NoError(t, err)
	require.Equal(t, "unavailable", report.DataState)
	payload := readBackupHealthTestPayload(t, report.Data)
	require.Equal(t, "unavailable", payload.Summary.HistoryState)
	require.Zero(t, payload.Summary.Counts.Total())
	require.Equal(t, "unavailable", payload.Summary.Workloads[0].HistoryState)
	require.Contains(t, payload.HistoryNotice, "no successful backups are assumed")
	h.SetCommercialLicenseResolver(func(context.Context) *licenseService { return newLicenseService() })
	_, failed := h.runReportSchedule(ctx, config.NewConfigPersistence(t.TempDir()), s, time.Now(), true, "")
	require.Equal(t, "failed", failed.LastRunStatus)
	require.Contains(t, failed.LastError, "entitlement required")
}

func TestBackupHealthScopeCannotIncludeAnotherClientsTasks(t *testing.T) {
	h, _, ctx, _, _ := inventoryScheduleFixture(t)
	snapshot, ok := h.getReportingEnrichmentSnapshot(ctx, "default")
	require.True(t, ok)
	require.Greater(t, len(snapshot.Resources), 1)
	snapshot.Resources[1].Tags = []string{"client-b"}
	scope := config.ReportScheduleScope{Resources: []config.ReportScheduleResource{{ResourceType: "vm", ResourceID: snapshot.Resources[0].ID}}, Tags: []string{"client-b"}}
	workloads, err := backupHealthWorkloads(snapshot, scope)
	require.NoError(t, err)
	require.Len(t, workloads, 2)
	workloads, err = backupHealthWorkloads(snapshot, config.ReportScheduleScope{Resources: []config.ReportScheduleResource{{ResourceType: "vm", ResourceID: "other-org-resource"}}})
	require.NoError(t, err)
	require.Empty(t, workloads)
	_, err = backupHealthWorkloads(snapshot, config.ReportScheduleScope{Resources: []config.ReportScheduleResource{{ResourceType: "host", ResourceID: "host"}}})
	require.Error(t, err)
	otherOrg := context.WithValue(ctx, OrgIDContextKey, "unconnected-organization")
	_, err = h.generateBackupHealthReport(otherOrg, config.ReportSchedule{}, time.Now(), true)
	require.ErrorContains(t, err, "scope is unavailable")
	bad := validReportSchedulePayload()
	bad.ID = "backup-scope"
	bad.Kind = config.ReportScheduleKindBackupHealth
	bad.Scope = config.ReportScheduleScope{Tags: []string{"client\nforged"}}
	_, err = h.prepareReportSchedule(ctx, bad, time.Now(), false)
	require.ErrorContains(t, err, "control characters")
	container := snapshot.Resources[0]
	container.ID, container.Type = "container-1", unifiedresources.ResourceTypeSystemContainer
	snapshot.Resources = append(snapshot.Resources, container)
	workloads, err = backupHealthWorkloads(snapshot, config.ReportScheduleScope{Resources: []config.ReportScheduleResource{{ResourceID: container.ID, ResourceType: "system-container"}}})
	require.NoError(t, err)
	require.Len(t, workloads, 1)
	require.Equal(t, "system-container", workloads[0].Type)
}

func TestBackupHealthRetriesKeepWindowAndExposeDataState(t *testing.T) {
	installBackupHealthTestRenderer(t)
	h, p, ctx, s, _ := inventoryScheduleFixture(t)
	s.Kind = config.ReportScheduleKindBackupHealth
	cfg, attempts := reportSMTP(t, []int{451, 250}, nil)
	require.NoError(t, p.SaveEmailConfig(cfg))
	s.Delivery = config.ReportScheduleDelivery{Method: "email", Attach: true, SaveToDisk: true}
	require.NoError(t, p.SaveReportScheduleStore(config.ReportScheduleStore{Schedules: []config.ReportSchedule{s}}))
	now := time.Now().UTC()
	// Drive an explicit manual attempt so this test does not depend on weekday.
	store := config.ReportScheduleStore{Schedules: []config.ReportSchedule{s}}
	_, first, err := h.executeReportSchedule(ctx, p, &store, 0, now, true, "", false)
	require.NoError(t, err)
	require.Equal(t, "retrying", first.LastRunStatus)
	require.Equal(t, "unavailable", first.LastReportDataState)
	require.NotNil(t, first.RetryAt)
	firstData, err := os.ReadFile(first.LastReportPath)
	require.NoError(t, err)
	restarted := NewReportingHandlers(h.reportMonitorManager(), nil)
	restarted.runDueReportSchedulesForOrg(ctx, "default", first.RetryAt.Add(time.Second))
	finished := storedReport(t, p)
	require.Equal(t, "ok", finished.LastRunStatus, finished.LastError)
	require.Equal(t, "unavailable", finished.LastReportDataState)
	require.EqualValues(t, 2, attempts.Load())
	secondData, err := os.ReadFile(finished.LastReportPath)
	require.NoError(t, err)
	firstPayload := readBackupHealthTestPayload(t, firstData)
	secondPayload := readBackupHealthTestPayload(t, secondData)
	require.Equal(t, firstPayload.Summary.Start, secondPayload.Summary.Start)
	require.Equal(t, firstPayload.Summary.End, secondPayload.Summary.End)
}