Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package api
import (
"bytes"
"context"
"encoding/json"
"errors"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"time"
"unsafe"
"github.com/rcourtman/pulse-go-rewrite/internal/ai"
"github.com/rcourtman/pulse-go-rewrite/internal/config"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/monitoring"
"github.com/rcourtman/pulse-go-rewrite/internal/notifications"
"github.com/rcourtman/pulse-go-rewrite/internal/securityutil"
"github.com/rcourtman/pulse-go-rewrite/pkg/reporting"
)
func newTestReportingScheduleHandlers(t *testing.T) (*ReportingHandlers, *config.ConfigPersistence) {
t.Helper()
baseDir := t.TempDir()
mtp := config.NewMultiTenantPersistence(baseDir)
mtm := monitoring.NewMultiTenantMonitor(&config.Config{DataPath: baseDir}, mtp, nil)
t.Cleanup(mtm.Stop)
monitor, err := mtm.GetMonitor("default")
if err != nil {
t.Fatalf("GetMonitor: %v", err)
}
persistence := monitor.GetConfigPersistence()
if persistence == nil {
t.Fatal("monitor config persistence is nil")
}
// A schedule only reports on resources its organization holds, so the
// default organization holds the VM validReportSchedulePayload names.
stateField := reflect.ValueOf(monitor).Elem().FieldByName("state")
state := reflect.NewAt(stateField.Type(), unsafe.Pointer(stateField.UnsafeAddr())).Elem().Interface().(*models.State)
state.UpdateVMs([]models.VM{{ID: "vm-1", Name: "VM 1", Status: "running"}})
return NewReportingHandlers(mtm, nil), persistence
}
func TestRunReportScheduleRequiresCurrentAdvancedReportingEntitlement(t *testing.T) {
handler := NewReportingHandlers(nil, nil)
service := newLicenseService()
handler.SetCommercialLicenseResolver(func(context.Context) *licenseService { return service })
now := time.Date(2026, 7, 14, 10, 0, 0, 0, time.UTC)
schedule := config.NormalizeReportSchedule(config.ReportSchedule{
ID: "report-schedule-entitlement",
Name: "Entitlement proof",
Enabled: true,
})
result, updated := handler.runReportSchedule(context.Background(), nil, schedule, now, false, "weekly:proof")
if result.path != "" || result.email != "" {
t.Fatalf("unentitled scheduler generated output: %+v", result)
}
if updated.LastRunStatus != config.ReportScheduleLastRunFailed || !strings.Contains(updated.LastError, "advanced reporting entitlement required") {
t.Fatalf("unexpected unentitled result: %+v", updated)
}
}
func TestPurgeGeneratedReportsAfterDowngradePreservesDefinitionsAndRejectsSymlinks(t *testing.T) {
baseDir := t.TempDir()
persistence := config.NewConfigPersistence(baseDir)
schedule := config.NormalizeReportSchedule(config.ReportSchedule{ID: "report-schedule-purge", Name: "Purge proof"})
report := generatedMultiReport{Filename: "report.pdf", Format: reporting.FormatPDF, Data: []byte("report")}
path, err := saveGeneratedReport(persistence, schedule, report)
if err != nil {
t.Fatalf("save report: %v", err)
}
outside := filepath.Join(t.TempDir(), "outside.pdf")
if err := os.WriteFile(outside, []byte("keep"), 0o600); err != nil {
t.Fatalf("write outside: %v", err)
}
if err := os.Symlink(outside, filepath.Join(filepath.Dir(path), "outside-link.pdf")); err != nil {
t.Fatalf("symlink: %v", err)
}
if err := purgeGeneratedReportsAfterDowngrade(persistence); err != nil {
t.Fatalf("purge: %v", err)
}
if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) {
t.Fatalf("generated report still exists: %v", err)
}
if data, err := os.ReadFile(outside); err != nil || string(data) != "keep" {
t.Fatalf("outside target changed: data=%q err=%v", data, err)
}
}
func validReportSchedulePayload() config.ReportSchedule {
return config.ReportSchedule{
Name: "Monthly client report",
Enabled: true,
Cadence: config.ReportScheduleCadence{
Type: config.ReportScheduleCadenceMonthly,
DayOfMonth: 1,
Time: "09:00",
Timezone: "UTC",
},
Scope: config.ReportScheduleScope{
Resources: []config.ReportScheduleResource{{ResourceType: "vm", ResourceID: "vm-1", Name: "VM 1"}},
},
Format: config.ReportScheduleFormatPDF,
Delivery: config.ReportScheduleDelivery{
Method: config.ReportScheduleDeliveryDisk,
Attach: false,
SaveToDisk: true,
},
RetentionCount: 12,
}
}
func encodeScheduleBody(t *testing.T, schedule config.ReportSchedule) *bytes.Reader {
t.Helper()
data, err := json.Marshal(schedule)
if err != nil {
t.Fatalf("marshal schedule: %v", err)
}
return bytes.NewReader(data)
}
func TestReportScheduleHandlersPersistCRUD(t *testing.T) {
handlers, persistence := newTestReportingScheduleHandlers(t)
ctx := context.WithValue(context.Background(), OrgIDContextKey, "default")
createReq := httptest.NewRequest(http.MethodPost, "/api/admin/reports/schedules", encodeScheduleBody(t, validReportSchedulePayload())).WithContext(ctx)
createRec := httptest.NewRecorder()
handlers.HandleCreateReportSchedule(createRec, createReq)
if createRec.Code != http.StatusCreated {
t.Fatalf("create status = %d, want %d body=%q", createRec.Code, http.StatusCreated, createRec.Body.String())
}
var created config.ReportSchedule
if err := json.NewDecoder(createRec.Body).Decode(&created); err != nil {
t.Fatalf("decode create: %v", err)
}
if created.ID == "" {
t.Fatal("created schedule ID is empty")
}
stored, err := persistence.LoadReportScheduleStore()
if err != nil {
t.Fatalf("LoadReportScheduleStore: %v", err)
}
if len(stored.Schedules) != 1 || stored.Schedules[0].ID != created.ID {
t.Fatalf("stored schedules = %+v, want created schedule", stored.Schedules)
}
listReq := httptest.NewRequest(http.MethodGet, "/api/admin/reports/schedules", nil).WithContext(ctx)
listRec := httptest.NewRecorder()
handlers.HandleListReportSchedules(listRec, listReq)
if listRec.Code != http.StatusOK {
t.Fatalf("list status = %d, want %d body=%q", listRec.Code, http.StatusOK, listRec.Body.String())
}
var list reportScheduleListResponse
if err := json.NewDecoder(listRec.Body).Decode(&list); err != nil {
t.Fatalf("decode list: %v", err)
}
if len(list.Schedules) != 1 || list.Schedules[0].ID != created.ID {
t.Fatalf("list schedules = %+v, want created schedule", list.Schedules)
}
created.Enabled = false
created.Name = "Updated monthly report"
updateReq := httptest.NewRequest(http.MethodPut, "/api/admin/reports/schedules/"+created.ID, encodeScheduleBody(t, created)).WithContext(ctx)
updateReq.SetPathValue("id", created.ID)
updateRec := httptest.NewRecorder()
handlers.HandleUpdateReportSchedule(updateRec, updateReq)
if updateRec.Code != http.StatusOK {
t.Fatalf("update status = %d, want %d body=%q", updateRec.Code, http.StatusOK, updateRec.Body.String())
}
var updated config.ReportSchedule
if err := json.NewDecoder(updateRec.Body).Decode(&updated); err != nil {
t.Fatalf("decode update: %v", err)
}
if updated.Enabled || updated.Name != "Updated monthly report" {
t.Fatalf("updated schedule = %+v, want disabled updated name", updated)
}
deleteReq := httptest.NewRequest(http.MethodDelete, "/api/admin/reports/schedules/"+created.ID, nil).WithContext(ctx)
deleteReq.SetPathValue("id", created.ID)
deleteRec := httptest.NewRecorder()
handlers.HandleDeleteReportSchedule(deleteRec, deleteReq)
if deleteRec.Code != http.StatusNoContent {
t.Fatalf("delete status = %d, want %d body=%q", deleteRec.Code, http.StatusNoContent, deleteRec.Body.String())
}
stored, err = persistence.LoadReportScheduleStore()
if err != nil {
t.Fatalf("LoadReportScheduleStore after delete: %v", err)
}
if len(stored.Schedules) != 0 {
t.Fatalf("stored schedules after delete = %+v, want empty", stored.Schedules)
}
}
func TestRunReportSchedulePersistsFailureStatusWhenEngineUnavailable(t *testing.T) {
original := reporting.GetEngine()
reporting.SetEngine(nil)
t.Cleanup(func() { reporting.SetEngine(original) })
handlers, persistence := newTestReportingScheduleHandlers(t)
ctx := context.WithValue(context.Background(), OrgIDContextKey, "default")
schedule := validReportSchedulePayload()
schedule.ID = "schedule-run-failure"
if err := persistence.SaveReportScheduleStore(config.ReportScheduleStore{Schedules: []config.ReportSchedule{schedule}}); err != nil {
t.Fatalf("SaveReportScheduleStore: %v", err)
}
req := httptest.NewRequest(http.MethodPost, "/api/admin/reports/schedules/schedule-run-failure/run", nil).WithContext(ctx)
req.SetPathValue("id", "schedule-run-failure")
rec := httptest.NewRecorder()
handlers.HandleRunReportSchedule(rec, req)
if rec.Code != http.StatusInternalServerError {
t.Fatalf("run status = %d, want %d body=%q", rec.Code, http.StatusInternalServerError, rec.Body.String())
}
stored, err := persistence.LoadReportScheduleStore()
if err != nil {
t.Fatalf("LoadReportScheduleStore: %v", err)
}
if len(stored.Schedules) != 1 {
t.Fatalf("stored schedule len = %d, want 1", len(stored.Schedules))
}
if stored.Schedules[0].LastRunStatus != config.ReportScheduleLastRunFailed {
t.Fatalf("last_run_status = %q, want failed", stored.Schedules[0].LastRunStatus)
}
if stored.Schedules[0].LastRunAt == nil || stored.Schedules[0].LastError == "" {
t.Fatalf("run status fields = lastRunAt %v lastError %q, want populated", stored.Schedules[0].LastRunAt, stored.Schedules[0].LastError)
}
}
func TestSaveGeneratedReportUsesHashedScheduleDirectory(t *testing.T) {
_, persistence := newTestReportingScheduleHandlers(t)
schedule := validReportSchedulePayload()
schedule.ID = "../tenant-owned-schedule"
report := generatedMultiReport{
Data: []byte("report"),
Filename: "../unsafe-report.pdf",
Format: reporting.FormatPDF,
}
path, err := saveGeneratedReport(persistence, schedule, report)
if err != nil {
t.Fatalf("saveGeneratedReport() error = %v", err)
}
wantDir := filepath.Join(
persistence.DataDir(),
"reports",
"generated",
securityutil.HashedStorageName(schedule.ID),
)
if filepath.Dir(path) != wantDir {
t.Fatalf("report dir = %q, want %q", filepath.Dir(path), wantDir)
}
if strings.Contains(path, schedule.ID) {
t.Fatalf("report path leaked raw schedule ID: %q", path)
}
if strings.ContainsAny(filepath.Base(path), `/\`) {
t.Fatalf("report filename still contains a path separator: %q", filepath.Base(path))
}
}
func TestPreparePatrolDigestScheduleFixesFormatScopeAndDelivery(t *testing.T) {
handlers, persistence := newTestReportingScheduleHandlers(t)
if err := persistence.SaveEmailConfig(notifications.EmailConfig{Enabled: true, SMTPHost: "smtp.example.test", SMTPPort: 25, From: "[email protected]", To: []string{"[email protected]"}}); err != nil {
t.Fatal(err)
}
ctx := context.WithValue(context.Background(), OrgIDContextKey, "default")
now := time.Date(2026, 9, 2, 10, 0, 0, 0, time.UTC)
base := config.ReportSchedule{
ID: "digest-1",
Name: "Patrol weekly summary",
Enabled: true,
Kind: config.ReportScheduleKindPatrolDigest,
Cadence: config.ReportScheduleCadence{Type: config.ReportScheduleCadenceWeekly, Weekday: "monday", Time: "08:00", Timezone: "UTC"},
// Clients may still send resource-report fields; the digest kind ignores them.
Scope: config.ReportScheduleScope{Tags: []string{"production"}},
Format: config.ReportScheduleFormatPDF,
Delivery: config.ReportScheduleDelivery{Method: config.ReportScheduleDeliveryEmail, To: []string{"[email protected]"}, Attach: true, SaveToDisk: true},
}
prepared, err := handlers.prepareReportSchedule(ctx, base, now, false)
if err != nil {
t.Fatalf("prepare digest schedule: %v", err)
}
if prepared.Format != config.ReportScheduleFormatEmail || prepared.Delivery.Attach || prepared.Delivery.SaveToDisk {
t.Fatalf("digest schedule must be an email-only summary, got %+v", prepared)
}
if len(prepared.Scope.Resources) != 0 || len(prepared.Scope.Tags) != 0 {
t.Fatalf("digest schedule must not carry a resource scope, got %+v", prepared.Scope)
}
if prepared.NextRunAt == nil || !prepared.NextRunAt.After(now) {
t.Fatalf("digest schedule needs a next run, got %v", prepared.NextRunAt)
}
monthly := base
monthly.Cadence = config.ReportScheduleCadence{Type: config.ReportScheduleCadenceMonthly, DayOfMonth: 1, Time: "08:00", Timezone: "UTC"}
if _, err := handlers.prepareReportSchedule(ctx, monthly, now, false); err == nil || !strings.Contains(err.Error(), "weekly") {
t.Fatalf("monthly digest must be rejected, got %v", err)
}
disk := base
disk.Delivery.Method = config.ReportScheduleDeliveryDisk
if _, err := handlers.prepareReportSchedule(ctx, disk, now, false); err == nil || !strings.Contains(err.Error(), "email") {
t.Fatalf("disk digest must be rejected, got %v", err)
}
unknown := base
unknown.Kind = "spreadsheet"
if _, err := handlers.prepareReportSchedule(ctx, unknown, now, false); err == nil || !strings.Contains(err.Error(), "kind") {
t.Fatalf("unknown kind must be rejected, got %v", err)
}
// Legacy schedules without a kind stay resource reports.
legacy := validReportSchedulePayload()
legacy.ID = "legacy-1"
preparedLegacy, err := handlers.prepareReportSchedule(ctx, legacy, now, false)
if err != nil {
t.Fatalf("prepare legacy schedule: %v", err)
}
if preparedLegacy.Kind != config.ReportScheduleKindResources || preparedLegacy.Format != config.ReportScheduleFormatPDF {
t.Fatalf("legacy schedule = %+v, want resources kind", preparedLegacy)
}
}
func TestRunPatrolDigestScheduleFailsClearlyWithoutDigestOrEmail(t *testing.T) {
handlers, persistence := newTestReportingScheduleHandlers(t)
ctx := context.WithValue(context.Background(), OrgIDContextKey, "default")
now := time.Date(2026, 9, 2, 10, 0, 0, 0, time.UTC)
schedule := config.NormalizeReportSchedule(config.ReportSchedule{
ID: "digest-run", Name: "Patrol weekly summary", Enabled: true, Kind: config.ReportScheduleKindPatrolDigest,
Cadence: config.ReportScheduleCadence{Type: config.ReportScheduleCadenceWeekly, Weekday: "monday", Time: "08:00", Timezone: "UTC"},
Format: config.ReportScheduleFormatEmail,
Delivery: config.ReportScheduleDelivery{Method: config.ReportScheduleDeliveryEmail},
})
_, withoutResolver := handlers.runReportSchedule(ctx, persistence, schedule, now, true, "")
if withoutResolver.LastRunStatus != config.ReportScheduleLastRunFailed || !strings.Contains(withoutResolver.LastError, "unavailable") {
t.Fatalf("run without resolver = %+v, want a clear unavailable failure", withoutResolver)
}
handlers.SetPatrolDigestResolver(func(context.Context, int) (ai.PatrolDigest, bool) { return ai.PatrolDigest{}, false })
_, notAvailable := handlers.runReportSchedule(ctx, persistence, schedule, now, true, "")
if notAvailable.LastRunStatus != config.ReportScheduleLastRunFailed || !strings.Contains(notAvailable.LastError, "not available") {
t.Fatalf("run without Patrol = %+v", notAvailable)
}
calls := 0
handlers.SetPatrolDigestResolver(func(_ context.Context, days int) (ai.PatrolDigest, bool) {
calls++
if days != ai.PatrolDigestDefaultDays {
t.Fatalf("digest days = %d, want %d", days, ai.PatrolDigestDefaultDays)
}
return ai.BuildPatrolDigest(ai.PatrolDigestInput{Now: now, Days: days}), true
})
_, noEmail := handlers.runReportSchedule(ctx, persistence, schedule, now, true, "")
if calls != 1 {
t.Fatalf("resolver calls = %d, want 1", calls)
}
if noEmail.LastRunStatus != config.ReportScheduleLastRunFailed || !strings.Contains(noEmail.LastError, "Enable email notifications") {
t.Fatalf("run without email config = %+v, want a failure that names the missing email destination", noEmail)
}
if noEmail.NextRunAt == nil {
t.Fatal("failed digest run must still schedule the next occurrence")
}
}
// A schedule runs inside its owning organization: its scope resolves against
// that organization's inventory and its report reads only that organization's
// metrics store. The fixture lives in reporting_org_scope_test.go.
func TestReportingScheduledRunReadsOnlyItsOwningOrganizationsMetrics(t *testing.T) {
handlers := newReportingOrgScopeFixture(t)
unifiedA := orgScopeUnifiedNodeID(t, handlers, "org-a")
persistence := config.NewConfigPersistence(t.TempDir())
// A schedule saved in org B that names org A's resources alongside its
// own reports only on org B's resource, from org B's store.
schedule := validReportSchedulePayload()
schedule.ID = "org-b-weekly"
schedule.Cadence = config.ReportScheduleCadence{Type: config.ReportScheduleCadenceWeekly, Weekday: "monday", Time: "09:00", Timezone: "UTC"}
schedule.Format = config.ReportScheduleFormatCSV
schedule.Scope = config.ReportScheduleScope{Resources: []config.ReportScheduleResource{
{ResourceType: "node", ResourceID: "node-a"},
{ResourceType: "node", ResourceID: unifiedA},
{ResourceType: "node", ResourceID: "node-b"},
}}
now := time.Now()
result, updated := handlers.runReportSchedule(orgScopeContext("org-b"), persistence, schedule, now, true, "")
if updated.LastRunStatus != config.ReportScheduleLastRunOK || result.path == "" {
t.Fatalf("org B scheduled run = %+v (%q), want a saved report", updated, updated.LastError)
}
saved, err := os.ReadFile(filepath.Clean(result.path))
if err != nil {
t.Fatalf("read saved report: %v", err)
}
if body := string(saved); !strings.Contains(body, orgScopeValueB) || strings.Contains(body, orgScopeValueA) || strings.Contains(body, "node-a") {
t.Fatalf("org B scheduled report must carry only org B's resource and metrics:\n%s", body)
}
// A schedule whose resources all belong to another organization reads
// nothing.
schedule.Scope = config.ReportScheduleScope{Resources: []config.ReportScheduleResource{
{ResourceType: "node", ResourceID: "node-a"},
}}
result, updated = handlers.runReportSchedule(orgScopeContext("org-b"), persistence, schedule, now, true, "")
if updated.LastRunStatus != config.ReportScheduleLastRunFailed || result.path != "" {
t.Fatalf("org B schedule over org A's resource = %+v, want a failed run with no report", updated)
}
}