Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

14.3 KB
Raw Blame
package api

import (
	"context"
	"encoding/json"
	"net"
	"net/http"
	"net/http/httptest"
	"net/textproto"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/config"
	"github.com/rcourtman/pulse-go-rewrite/internal/notifications"
	"github.com/rcourtman/pulse-go-rewrite/pkg/reporting"
	"github.com/stretchr/testify/require"
)

func reportDeliveryFixture(t *testing.T) (*ReportingHandlers, *config.ConfigPersistence, *stubReportingEngine, context.Context, config.ReportSchedule) {
	t.Helper()
	h, p := newTestReportingScheduleHandlers(t)
	engine := &stubReportingEngine{data: []byte("cpu\n42\n"), contentType: "text/csv"}
	previous := reporting.GetEngine()
	reporting.SetEngine(engine)
	t.Cleanup(func() { reporting.SetEngine(previous) })
	s := validReportSchedulePayload()
	s.ID = "delivery"
	s.Cadence = config.ReportScheduleCadence{Type: "weekly", Weekday: "monday", Time: "09:00", Timezone: "UTC"}
	s.Format = "csv"
	s.Delivery = config.ReportScheduleDelivery{Method: "email", To: []string{"[email protected]"}, Attach: true, SaveToDisk: true}
	return h, p, engine, context.WithValue(context.Background(), OrgIDContextKey, "default"), s
}

// SMTP receives real report bytes and supplies structured completed-message
// verdicts. Zero drops the connection after DATA, making acceptance uncertain.
func reportSMTP(t *testing.T, replies []int, onMessage func()) (notifications.EmailConfig, *atomic.Int32) {
	t.Helper()
	listener, err := net.Listen("tcp4", "127.0.0.1:0")
	require.NoError(t, err)
	var sessions sync.WaitGroup
	attempts := &atomic.Int32{}
	done := make(chan struct{})
	go func() {
		defer close(done)
		for {
			wire, err := listener.Accept()
			if err != nil {
				return
			}
			sessions.Add(1)
			go func() {
				defer sessions.Done()
				defer wire.Close()
				_ = wire.SetDeadline(time.Now().Add(10 * time.Second))
				conn := textproto.NewConn(wire)
				if conn.PrintfLine("220 localhost") != nil {
					return
				}
				for {
					line, err := conn.ReadLine()
					if err != nil {
						return
					}
					switch strings.Fields(line)[0] {
					case "EHLO", "HELO", "MAIL", "RCPT":
						err = conn.PrintfLine("250 OK")
					case "DATA":
						if conn.PrintfLine("354 Send message") != nil {
							return
						}
						body, readErr := conn.ReadDotBytes()
						if readErr != nil {
							return
						}
						if !strings.Contains(string(body), "Content-Disposition: attachment") {
							t.Error("report has no MIME attachment")
						}
						attempt := int(attempts.Add(1)) - 1
						if onMessage != nil {
							onMessage()
						}
						code := replies[len(replies)-1]
						if attempt < len(replies) {
							code = replies[attempt]
						}
						if code == 0 {
							return
						}
						err = conn.PrintfLine("%d provider-verdict-do-not-persist", code)
					case "QUIT":
						_ = conn.PrintfLine("221 Bye")
						return
					default:
						t.Errorf("unexpected SMTP command %q", line)
						return
					}
					if err != nil {
						return
					}
				}
			}()
		}
	}()
	t.Cleanup(func() { _ = listener.Close(); <-done; sessions.Wait() })
	_, port, err := net.SplitHostPort(listener.Addr().String())
	require.NoError(t, err)
	n, err := strconv.Atoi(port)
	require.NoError(t, err)
	return notifications.EmailConfig{Enabled: true, SMTPHost: "127.0.0.1", SMTPPort: n, From: "[email protected]", To: []string{"[email protected]"}}, attempts
}

func storedReport(t *testing.T, p *config.ConfigPersistence) config.ReportSchedule {
	t.Helper()
	store, err := p.LoadReportScheduleStore()
	require.NoError(t, err)
	require.Len(t, store.Schedules, 1)
	return store.Schedules[0]
}

func runDeliveryRequest(h *ReportingHandlers, ctx context.Context, retry bool) *httptest.ResponseRecorder {
	url := "/api/admin/reports/schedules/delivery/run"
	if retry {
		url += "?retry=true"
	}
	r := httptest.NewRequest(http.MethodPost, url, nil).WithContext(ctx)
	r.SetPathValue("id", "delivery")
	w := httptest.NewRecorder()
	h.HandleRunReportSchedule(w, r)
	return w
}

func TestReportScheduleDeliveryPreflightAndServerOwnedState(t *testing.T) {
	h, p, engine, ctx, s := reportDeliveryFixture(t)
	now := time.Now().UTC()
	s.LastRunStatus = "ok"
	s.LastReportPath = "/untrusted/path"
	s.LastSuccessAt = &now
	s.RetryAt = &now
	s.RunWindowEnd = &now
	s.RunAttempts = 99
	r := httptest.NewRequest(http.MethodPost, "/api/admin/reports/schedules", encodeScheduleBody(t, s)).WithContext(ctx)
	w := httptest.NewRecorder()
	h.HandleCreateReportSchedule(w, r)
	require.Equal(t, http.StatusBadRequest, w.Code, w.Body.String())
	require.Contains(t, w.Body.String(), "Enable email notifications")
	s.Enabled = false
	r = httptest.NewRequest(http.MethodPost, "/api/admin/reports/schedules", encodeScheduleBody(t, s)).WithContext(ctx)
	w = httptest.NewRecorder()
	h.HandleCreateReportSchedule(w, r)
	require.Equal(t, http.StatusCreated, w.Code, w.Body.String())
	created := storedReport(t, p)
	require.Empty(t, created.LastRunStatus)
	require.Empty(t, created.LastReportPath)
	require.Nil(t, created.LastSuccessAt)
	require.Nil(t, created.RetryAt)
	require.Nil(t, created.RunWindowEnd)
	require.Zero(t, created.RunAttempts)
	created.ID = "delivery"
	require.NoError(t, p.SaveReportScheduleStore(config.ReportScheduleStore{Schedules: []config.ReportSchedule{created}}))
	w = runDeliveryRequest(h, ctx, false)
	require.Equal(t, http.StatusInternalServerError, w.Code, w.Body.String())
	require.Empty(t, engine.lastMulti.Resources, "missing email config must fail before generation")
	failed := storedReport(t, p)
	require.Equal(t, "failed", failed.LastRunStatus)
	require.Nil(t, failed.RetryAt)
	require.Empty(t, failed.LastReportPath)
	list := httptest.NewRecorder()
	h.HandleListReportSchedules(list, httptest.NewRequest(http.MethodGet, "/", nil).WithContext(ctx))
	var payload reportScheduleListResponse
	require.NoError(t, json.Unmarshal(list.Body.Bytes(), &payload))
	require.False(t, payload.DeliveryChecks["delivery"].Ready)
}

func TestReportScheduleDeliveryRetriesSurviveRestartAndKeepWindow(t *testing.T) {
	h, p, engine, ctx, s := reportDeliveryFixture(t)
	cfg, attempts := reportSMTP(t, []int{451, 451, 250}, nil)
	require.NoError(t, p.SaveEmailConfig(cfg))
	require.NoError(t, p.SaveReportScheduleStore(config.ReportScheduleStore{Schedules: []config.ReportSchedule{s}}))
	now := time.Date(2026, 10, 19, 9, 0, 1, 0, time.UTC)
	h.runDueReportSchedulesForOrg(ctx, "default", now)
	first := storedReport(t, p)
	require.Equal(t, "retrying", first.LastRunStatus)
	require.Equal(t, 1, first.RunAttempts)
	require.NotNil(t, first.RetryAt)
	window := engine.lastMulti.End
	require.NotContains(t, first.LastError, "provider-verdict")
	// A fresh handler reads only the persisted state, with no in-memory retry job.
	restarted := NewReportingHandlers(h.reportMonitorManager(), nil)
	restarted.runDueReportSchedulesForOrg(ctx, "default", now.Add(30*time.Second))
	require.EqualValues(t, 1, attempts.Load())
	restarted.runDueReportSchedulesForOrg(ctx, "default", first.RetryAt.Add(time.Second))
	second := storedReport(t, p)
	require.Equal(t, "retrying", second.LastRunStatus)
	require.Equal(t, 2, second.RunAttempts)
	require.True(t, second.RetryAt.After(first.RetryAt.Add(4*time.Minute)))
	restarted.runDueReportSchedulesForOrg(ctx, "default", second.RetryAt.Add(time.Second))
	finished := storedReport(t, p)
	require.Equal(t, "ok", finished.LastRunStatus)
	require.Equal(t, "sent_with_attachment", finished.LastDeliveryStatus)
	require.Equal(t, 3, finished.RunAttempts)
	require.Nil(t, finished.RetryAt)
	require.NotNil(t, finished.LastSuccessAt)
	require.FileExists(t, finished.LastReportPath)
	require.Equal(t, window, engine.lastMulti.End)
	restarted.runDueReportSchedulesForOrg(ctx, "default", now.Add(time.Hour))
	require.EqualValues(t, 3, attempts.Load())
}

func TestReportScheduleDeliveryStopsAtBudgetRejectionAndUncertainty(t *testing.T) {
	for _, tc := range []struct {
		name     string
		code     int
		want     string
		attempts int32
	}{{"exhausted", 451, "failed", 3}, {"rejected", 550, "failed", 1}, {"unconfirmed", 0, "unconfirmed", 1}} {
		t.Run(tc.name, func(t *testing.T) {
			h, p, _, ctx, s := reportDeliveryFixture(t)
			cfg, attempts := reportSMTP(t, []int{tc.code}, nil)
			require.NoError(t, p.SaveEmailConfig(cfg))
			require.NoError(t, p.SaveReportScheduleStore(config.ReportScheduleStore{Schedules: []config.ReportSchedule{s}}))
			now := time.Date(2026, 10, 19, 9, 0, 1, 0, time.UTC)
			for i := 0; i < 5; i++ {
				h.runDueReportSchedulesForOrg(ctx, "default", now.Add(time.Duration(i)*10*time.Minute))
			}
			out := storedReport(t, p)
			require.Equal(t, tc.want, out.LastRunStatus)
			require.Nil(t, out.RetryAt)
			require.Equal(t, tc.attempts, attempts.Load())
			require.NotContains(t, out.LastError, "provider-verdict")
			if tc.code == 0 {
				require.Contains(t, out.LastError, "duplicate")
			}
		})
	}
}

func TestReportScheduleDeliveryRecoversInterruptedEmailWithoutResending(t *testing.T) {
	h, p, _, ctx, s := reportDeliveryFixture(t)
	cfg, attempts := reportSMTP(t, []int{250}, nil)
	require.NoError(t, p.SaveEmailConfig(cfg))
	now := time.Date(2026, 10, 19, 9, 0, 1, 0, time.UTC)
	s.LastRunStatus = "running"
	s.RunWindowEnd = &now
	s.RunAttempts = 1
	s.LastOccurrenceKey = currentReportScheduleOccurrenceKey(s, now)
	require.NoError(t, p.SaveReportScheduleStore(config.ReportScheduleStore{Schedules: []config.ReportSchedule{s}}))
	h.runDueReportSchedulesForOrg(ctx, "default", now.Add(time.Minute))
	require.Equal(t, "unconfirmed", storedReport(t, p).LastRunStatus)
	require.Zero(t, attempts.Load())
	w := runDeliveryRequest(h, ctx, true)
	require.Equal(t, http.StatusOK, w.Code, w.Body.String())
	require.EqualValues(t, 1, attempts.Load())
	require.Equal(t, "ok", storedReport(t, p).LastRunStatus)
}

func TestReportScheduleDeliveryEditCancelsRetryAndPreservesOutcome(t *testing.T) {
	h, p, _, ctx, s := reportDeliveryFixture(t)
	cfg, attempts := reportSMTP(t, []int{451}, nil)
	require.NoError(t, p.SaveEmailConfig(cfg))
	require.NoError(t, p.SaveReportScheduleStore(config.ReportScheduleStore{Schedules: []config.ReportSchedule{s}}))
	require.Equal(t, http.StatusAccepted, runDeliveryRequest(h, ctx, false).Code)
	pending := storedReport(t, p)
	lastRun := *pending.LastRunAt
	pending.Enabled = false
	pending.LastRunStatus = "ok"
	pending.LastReportPath = "/forged"
	pending.RunAttempts = 99
	r := httptest.NewRequest(http.MethodPut, "/", encodeScheduleBody(t, pending)).WithContext(ctx)
	r.SetPathValue("id", "delivery")
	w := httptest.NewRecorder()
	h.HandleUpdateReportSchedule(w, r)
	require.Equal(t, http.StatusOK, w.Code, w.Body.String())
	out := storedReport(t, p)
	require.Equal(t, "failed", out.LastRunStatus)
	require.Equal(t, lastRun, *out.LastRunAt)
	require.NotEqual(t, "/forged", out.LastReportPath)
	require.Equal(t, 1, out.RunAttempts)
	require.Nil(t, out.RetryAt)
	require.Nil(t, out.RunWindowEnd)
	h.runDueReportSchedulesForOrg(ctx, "default", time.Now().Add(time.Hour))
	require.EqualValues(t, 1, attempts.Load())
	require.Equal(t, http.StatusConflict, runDeliveryRequest(h, ctx, true).Code)
}

func TestReportScheduleDeliveryPersistsBeforeSMTPAndCannotResurrectDeletedSchedule(t *testing.T) {
	h, p, _, ctx, s := reportDeliveryFixture(t)
	entered, release := make(chan struct{}), make(chan struct{})
	var releaseOnce sync.Once
	unblock := func() { releaseOnce.Do(func() { close(release) }) }
	var observed atomic.Bool
	cfg, _ := reportSMTP(t, []int{250}, func() {
		store, err := p.LoadReportScheduleStore()
		if err == nil && len(store.Schedules) == 1 && store.Schedules[0].LastRunStatus == "running" {
			observed.Store(true)
		}
		close(entered)
		<-release
	})
	t.Cleanup(unblock)
	require.NoError(t, p.SaveEmailConfig(cfg))
	require.NoError(t, p.SaveReportScheduleStore(config.ReportScheduleStore{Schedules: []config.ReportSchedule{s}}))
	runDone := make(chan *httptest.ResponseRecorder, 1)
	go func() { runDone <- runDeliveryRequest(h, ctx, false) }()
	select {
	case <-entered:
	case result := <-runDone:
		t.Fatalf("report ended before SMTP: %d %s", result.Code, result.Body.String())
	case <-time.After(15 * time.Second):
		t.Fatal("report did not reach SMTP")
	}
	deleteDone := make(chan *httptest.ResponseRecorder, 1)
	go func() {
		r := httptest.NewRequest(http.MethodDelete, "/", nil).WithContext(ctx)
		r.SetPathValue("id", "delivery")
		w := httptest.NewRecorder()
		h.HandleDeleteReportSchedule(w, r)
		deleteDone <- w
	}()
	select {
	case <-deleteDone:
		t.Error("delete passed an active delivery")
	case <-time.After(30 * time.Millisecond):
	}
	unblock()
	require.Equal(t, http.StatusOK, (<-runDone).Code)
	require.Equal(t, http.StatusNoContent, (<-deleteDone).Code)
	require.True(t, observed.Load())
	store, err := p.LoadReportScheduleStore()
	require.NoError(t, err)
	require.Empty(t, store.Schedules)
}

func TestReportScheduleDeliveryOversizeAndDiskOutcomes(t *testing.T) {
	h, p, _, ctx, s := reportDeliveryFixture(t)
	cfg, attempts := reportSMTP(t, []int{250}, nil)
	require.NoError(t, p.SaveEmailConfig(cfg))
	_, err := sendScheduledReportEmail(p, s, generatedMultiReport{Data: make([]byte, reportScheduleAttachmentLimitBytes+1)}, "/owned/report.pdf")
	require.ErrorContains(t, err, "15 MiB")
	require.Zero(t, attempts.Load())
	s.Delivery.Method = "disk"
	require.NoError(t, p.SaveReportScheduleStore(config.ReportScheduleStore{Schedules: []config.ReportSchedule{s}}))
	require.Equal(t, http.StatusOK, runDeliveryRequest(h, ctx, false).Code)
	out := storedReport(t, p)
	require.Equal(t, "saved", out.LastDeliveryStatus)
	require.Equal(t, "ok", out.LastRunStatus)
	require.FileExists(t, out.LastReportPath)
	require.Zero(t, attempts.Load())
}

func TestReportScheduleDeliveryRechecksEntitlementBeforeRetry(t *testing.T) {
	h, p, _, ctx, s := reportDeliveryFixture(t)
	cfg, attempts := reportSMTP(t, []int{451}, nil)
	require.NoError(t, p.SaveEmailConfig(cfg))
	require.NoError(t, p.SaveReportScheduleStore(config.ReportScheduleStore{Schedules: []config.ReportSchedule{s}}))
	require.Equal(t, http.StatusAccepted, runDeliveryRequest(h, ctx, false).Code)
	pending := storedReport(t, p)
	service := newLicenseService()
	h.SetCommercialLicenseResolver(func(context.Context) *licenseService { return service })
	h.runDueReportSchedulesForOrg(ctx, "default", pending.RetryAt.Add(time.Second))
	out := storedReport(t, p)
	require.Equal(t, "failed", out.LastRunStatus)
	require.Contains(t, out.LastError, "advanced reporting entitlement required")
	require.Nil(t, out.RetryAt)
	require.EqualValues(t, 1, attempts.Load(), "expired entitlement must not dispatch another message")
}