Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package api
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"os"
"sort"
"strings"
"time"
"github.com/google/uuid"
"github.com/rcourtman/pulse-go-rewrite/internal/ai"
"github.com/rcourtman/pulse-go-rewrite/internal/config"
"github.com/rcourtman/pulse-go-rewrite/internal/notifications"
"github.com/rcourtman/pulse-go-rewrite/internal/securityutil"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
"github.com/rcourtman/pulse-go-rewrite/pkg/reporting"
"github.com/rs/zerolog/log"
)
const (
reportScheduleAttachmentLimitBytes = 15 * 1024 * 1024
reportScheduleTickInterval = time.Minute
reportScheduleMissedRunGrace = 24 * time.Hour
reportScheduleMaxAttempts = 3
)
type reportScheduleListResponse struct {
Schedules []config.ReportSchedule `json:"schedules"`
DeliveryChecks map[string]reportScheduleDeliveryCheck `json:"delivery_checks"`
}
type reportScheduleDeliveryCheck struct {
Ready bool `json:"ready"`
Message string `json:"message"`
}
type reportScheduleRunResponse struct {
Schedule config.ReportSchedule `json:"schedule"`
Status string `json:"status"`
Path string `json:"path,omitempty"`
Email string `json:"email,omitempty"`
}
func (h *ReportingHandlers) HandleListReportSchedules(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
persistence, store, err := h.reportSchedulePersistenceAndStore(r.Context())
if err != nil {
writeErrorResponse(w, http.StatusInternalServerError, "schedule_store_unavailable", "Report schedules are unavailable", nil)
return
}
checks := make(map[string]reportScheduleDeliveryCheck, len(store.Schedules))
for _, schedule := range store.Schedules {
checks[schedule.ID] = checkReportScheduleDelivery(persistence, schedule)
}
writeReportScheduleJSON(w, http.StatusOK, reportScheduleListResponse{Schedules: store.Schedules, DeliveryChecks: checks})
}
func (h *ReportingHandlers) HandleCreateReportSchedule(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
var incoming config.ReportSchedule
if err := decodeReportScheduleBody(w, r, &incoming); err != nil {
return
}
h.scheduleRunMu.Lock()
defer h.scheduleRunMu.Unlock()
persistence, store, err := h.reportSchedulePersistenceAndStore(r.Context())
if err != nil {
writeErrorResponse(w, http.StatusInternalServerError, "schedule_store_unavailable", "Report schedules are unavailable", nil)
return
}
now := time.Now().UTC()
incoming.ID = uuid.NewString()
incoming.CreatedAt = now
incoming.UpdatedAt = now
copyReportScheduleRunState(&incoming, config.ReportSchedule{})
schedule, err := h.prepareReportSchedule(r.Context(), incoming, now, false)
if err != nil {
writeReportScheduleValidationError(w, err)
return
}
schedule.LastOccurrenceKey = currentReportScheduleOccurrenceKey(schedule, now)
store.Schedules = append(store.Schedules, schedule)
if err := persistence.SaveReportScheduleStore(*store); err != nil {
writeErrorResponse(w, http.StatusInternalServerError, "schedule_save_failed", "Failed to save report schedule", nil)
return
}
writeReportScheduleJSON(w, http.StatusCreated, schedule)
}
func (h *ReportingHandlers) HandleUpdateReportSchedule(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPut {
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
scheduleID := strings.TrimSpace(r.PathValue("id"))
if scheduleID == "" {
writeErrorResponse(w, http.StatusBadRequest, "missing_schedule_id", "Schedule ID is required", nil)
return
}
var incoming config.ReportSchedule
if err := decodeReportScheduleBody(w, r, &incoming); err != nil {
return
}
h.scheduleRunMu.Lock()
defer h.scheduleRunMu.Unlock()
persistence, store, err := h.reportSchedulePersistenceAndStore(r.Context())
if err != nil {
writeErrorResponse(w, http.StatusInternalServerError, "schedule_store_unavailable", "Report schedules are unavailable", nil)
return
}
index := findReportScheduleIndex(store.Schedules, scheduleID)
if index < 0 {
writeErrorResponse(w, http.StatusNotFound, "schedule_not_found", "Report schedule not found", nil)
return
}
now := time.Now().UTC()
existing := store.Schedules[index]
incoming.ID = existing.ID
incoming.CreatedAt = existing.CreatedAt
incoming.UpdatedAt = now
copyReportScheduleRunState(&incoming, existing)
schedule, err := h.prepareReportSchedule(r.Context(), incoming, now, false)
if err != nil {
writeReportScheduleValidationError(w, err)
return
}
if schedule.RetryAt != nil {
schedule.LastRunStatus = config.ReportScheduleLastRunFailed
schedule.LastError = "Automatic retry cancelled because the schedule was edited. Run a new report when ready."
}
schedule.RetryAt = nil
schedule.RunWindowEnd = nil
schedule.RunManual = false
schedule.LastOccurrenceKey = currentReportScheduleOccurrenceKey(schedule, now)
store.Schedules[index] = schedule
if err := persistence.SaveReportScheduleStore(*store); err != nil {
writeErrorResponse(w, http.StatusInternalServerError, "schedule_save_failed", "Failed to save report schedule", nil)
return
}
writeReportScheduleJSON(w, http.StatusOK, schedule)
}
func (h *ReportingHandlers) HandleDeleteReportSchedule(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodDelete {
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
scheduleID := strings.TrimSpace(r.PathValue("id"))
if scheduleID == "" {
writeErrorResponse(w, http.StatusBadRequest, "missing_schedule_id", "Schedule ID is required", nil)
return
}
h.scheduleRunMu.Lock()
defer h.scheduleRunMu.Unlock()
persistence, store, err := h.reportSchedulePersistenceAndStore(r.Context())
if err != nil {
writeErrorResponse(w, http.StatusInternalServerError, "schedule_store_unavailable", "Report schedules are unavailable", nil)
return
}
index := findReportScheduleIndex(store.Schedules, scheduleID)
if index < 0 {
writeErrorResponse(w, http.StatusNotFound, "schedule_not_found", "Report schedule not found", nil)
return
}
store.Schedules = append(store.Schedules[:index], store.Schedules[index+1:]...)
if err := persistence.SaveReportScheduleStore(*store); err != nil {
writeErrorResponse(w, http.StatusInternalServerError, "schedule_save_failed", "Failed to save report schedule", nil)
return
}
w.WriteHeader(http.StatusNoContent)
}
func (h *ReportingHandlers) HandleRunReportSchedule(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
scheduleID := strings.TrimSpace(r.PathValue("id"))
if scheduleID == "" {
writeErrorResponse(w, http.StatusBadRequest, "missing_schedule_id", "Schedule ID is required", nil)
return
}
h.scheduleRunMu.Lock()
defer h.scheduleRunMu.Unlock()
persistence, store, err := h.reportSchedulePersistenceAndStore(r.Context())
if err != nil {
writeErrorResponse(w, http.StatusInternalServerError, "schedule_store_unavailable", "Report schedules are unavailable", nil)
return
}
index := findReportScheduleIndex(store.Schedules, scheduleID)
if index < 0 {
writeErrorResponse(w, http.StatusNotFound, "schedule_not_found", "Report schedule not found", nil)
return
}
schedule := store.Schedules[index]
if schedule.LastRunStatus == config.ReportScheduleLastRunRunning {
writeErrorResponse(w, http.StatusConflict, "schedule_running", "This report is already running. Refresh its status before trying again.", nil)
return
}
retry := r.URL.Query().Get("retry") == "true"
if retry && (schedule.RunWindowEnd == nil || !reportScheduleRetryableOutcome(schedule.LastRunStatus)) {
writeErrorResponse(w, http.StatusConflict, "retry_unavailable", "The original failed report window is unavailable. Run a new report.", nil)
return
}
result, updated, err := h.executeReportSchedule(r.Context(), persistence, store, index, time.Now().UTC(), true, "", retry)
if err != nil {
writeErrorResponse(w, http.StatusInternalServerError, "schedule_save_failed", "The report outcome could not be saved. Check its status and the recipient before running again.", nil)
return
}
status := http.StatusOK
if updated.LastRunStatus == config.ReportScheduleLastRunFailed {
status = http.StatusInternalServerError
} else if updated.LastRunStatus == config.ReportScheduleLastRunRetrying {
status = http.StatusAccepted
} else if updated.LastRunStatus == config.ReportScheduleLastRunUnconfirmed {
status = http.StatusBadGateway
}
writeReportScheduleJSON(w, status, reportScheduleRunResponse{
Schedule: updated,
Status: updated.LastRunStatus,
Path: result.path,
Email: result.email,
})
}
func decodeReportScheduleBody(w http.ResponseWriter, r *http.Request, target *config.ReportSchedule) error {
r.Body = http.MaxBytesReader(w, r.Body, reportingMultiReportBodyMax)
decoder := json.NewDecoder(r.Body)
decoder.DisallowUnknownFields()
if err := decoder.Decode(target); err != nil {
writeErrorResponse(w, http.StatusBadRequest, "invalid_body", "Invalid report schedule body", nil)
return err
}
if err := decoder.Decode(&struct{}{}); err != nil && !errors.Is(err, io.EOF) {
writeErrorResponse(w, http.StatusBadRequest, "invalid_body", "Invalid report schedule body", nil)
return err
}
return nil
}
func writeReportScheduleJSON(w http.ResponseWriter, status int, payload any) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
_ = json.NewEncoder(w).Encode(payload)
}
type reportScheduleValidationError struct {
code string
message string
}
func (e reportScheduleValidationError) Error() string {
return e.message
}
func writeReportScheduleValidationError(w http.ResponseWriter, err error) {
var validationErr reportScheduleValidationError
if errors.As(err, &validationErr) {
writeErrorResponse(w, http.StatusBadRequest, validationErr.code, validationErr.message, nil)
return
}
writeErrorResponse(w, http.StatusBadRequest, "invalid_schedule", err.Error(), nil)
}
func (h *ReportingHandlers) reportSchedulePersistenceAndStore(ctx context.Context) (*config.ConfigPersistence, *config.ReportScheduleStore, error) {
persistence, err := h.reportSchedulePersistence(ctx)
if err != nil {
return nil, nil, err
}
store, err := persistence.LoadReportScheduleStore()
if err != nil {
return nil, nil, err
}
if store == nil {
empty := config.EmptyReportScheduleStore()
store = &empty
}
return persistence, store, nil
}
func (h *ReportingHandlers) reportSchedulePersistence(ctx context.Context) (*config.ConfigPersistence, error) {
manager := h.reportMonitorManager()
if manager == nil {
return nil, fmt.Errorf("multi-tenant monitor is not configured")
}
orgID := GetOrgID(ctx)
if strings.TrimSpace(orgID) == "" {
orgID = "default"
}
monitor, err := manager.GetMonitor(orgID)
if err != nil {
return nil, err
}
persistence := monitor.GetConfigPersistence()
if persistence == nil {
return nil, fmt.Errorf("config persistence is not configured")
}
return persistence, nil
}
func findReportScheduleIndex(schedules []config.ReportSchedule, id string) int {
for i := range schedules {
if schedules[i].ID == id {
return i
}
}
return -1
}
// Outcomes are server-owned even when older clients PUT a whole schedule.
func copyReportScheduleRunState(dst *config.ReportSchedule, src config.ReportSchedule) {
dst.LastRunAt = src.LastRunAt
dst.LastRunStatus = src.LastRunStatus
dst.LastError = src.LastError
dst.LastDeliveryStatus = src.LastDeliveryStatus
dst.LastReportPath = src.LastReportPath
dst.LastReportDataState = src.LastReportDataState
dst.LastReportDataMessage = src.LastReportDataMessage
dst.LastSuccessAt = src.LastSuccessAt
dst.RunAttempts = src.RunAttempts
dst.RetryAt = src.RetryAt
dst.RunWindowEnd = src.RunWindowEnd
dst.RunManual = src.RunManual
dst.NextRunAt = src.NextRunAt
dst.LastOccurrenceKey = src.LastOccurrenceKey
}
func currentReportScheduleOccurrenceKey(schedule config.ReportSchedule, now time.Time) string {
_, key, _ := lastReportScheduleOccurrenceAt(schedule, now)
return key
}
func checkReportScheduleDelivery(persistence *config.ConfigPersistence, schedule config.ReportSchedule) reportScheduleDeliveryCheck {
if schedule.Delivery.Method == config.ReportScheduleDeliveryDisk {
return reportScheduleDeliveryCheck{Ready: true, Message: "Reports are saved on this Pulse server."}
}
_, err := scheduledReportEmailManager(persistence, schedule)
if err != nil {
return reportScheduleDeliveryCheck{Message: err.Error()}
}
return reportScheduleDeliveryCheck{Ready: true, Message: "Email configuration checked. Run the report to test delivery."}
}
func (h *ReportingHandlers) validateEnabledReportDelivery(ctx context.Context, schedule config.ReportSchedule) error {
if !schedule.Enabled || schedule.Delivery.Method != config.ReportScheduleDeliveryEmail {
return nil
}
persistence, err := h.reportSchedulePersistence(ctx)
if err != nil {
return reportScheduleValidationError{code: "email_configuration_unavailable", message: "Email configuration is unavailable. Check Alerts > Notifications."}
}
check := checkReportScheduleDelivery(persistence, schedule)
if !check.Ready {
return reportScheduleValidationError{code: "email_not_ready", message: check.Message}
}
return nil
}
func reportScheduleRetryableOutcome(status string) bool {
return status == config.ReportScheduleLastRunFailed || status == config.ReportScheduleLastRunRetrying || status == config.ReportScheduleLastRunUnconfirmed
}
// The caller owns scheduleRunMu across load, side effect and final persistence.
// Persisting running first makes a lost completion record visible after restart.
func (h *ReportingHandlers) executeReportSchedule(ctx context.Context, persistence *config.ConfigPersistence, store *config.ReportScheduleStore, index int, now time.Time, manual bool, key string, retry bool) (reportScheduleRunResult, config.ReportSchedule, error) {
schedule := config.NormalizeReportSchedule(store.Schedules[index])
if !retry {
windowEnd := now.UTC().Truncate(time.Second)
schedule.RunWindowEnd = &windowEnd
schedule.RunManual = manual
if manual && key == "" {
key = currentReportScheduleOccurrenceKey(schedule, now)
}
}
// An explicit operator retry starts a fresh bounded delivery budget.
if !retry || manual {
schedule.RunAttempts = 0
}
schedule.RunAttempts++
schedule.LastRunStatus = config.ReportScheduleLastRunRunning
schedule.LastDeliveryStatus = ""
schedule.LastReportDataState = ""
schedule.LastReportDataMessage = ""
schedule.LastError = ""
schedule.RetryAt = nil
schedule.LastRunAt = &now
schedule.UpdatedAt = now
if key != "" {
schedule.LastOccurrenceKey = key
}
if next, err := nextReportScheduleRunAt(schedule, now); err == nil {
schedule.NextRunAt = &next
}
store.Schedules[index] = schedule
if err := persistence.SaveReportScheduleStore(*store); err != nil {
return reportScheduleRunResult{}, schedule, err
}
started := time.Now()
result, updated := h.runReportSchedule(ctx, persistence, schedule, now, schedule.RunManual, "")
completed := now.Add(time.Since(started)).UTC()
updated.UpdatedAt = completed
if updated.RetryAt != nil {
retryAt := completed.Add(updated.RetryAt.Sub(now))
updated.RetryAt = &retryAt
}
if updated.LastRunStatus == config.ReportScheduleLastRunOK {
updated.LastSuccessAt = &completed
}
store.Schedules[index] = updated
if err := persistence.SaveReportScheduleStore(*store); err != nil {
return result, updated, err
}
h.recordReportingSuccessfulPeriod(ctx, updated, result.usagePeriod, completed)
return result, updated, nil
}
func (h *ReportingHandlers) prepareReportSchedule(ctx context.Context, schedule config.ReportSchedule, now time.Time, statusOnly bool) (config.ReportSchedule, error) {
schedule = config.NormalizeReportSchedule(schedule)
if schedule.ID == "" {
return schedule, reportScheduleValidationError{code: "invalid_schedule_id", message: "Schedule ID is required"}
}
if statusOnly {
return schedule, nil
}
if schedule.Name == "" || len(schedule.Name) > 80 {
return schedule, reportScheduleValidationError{code: "invalid_name", message: "Schedule name is required and must be 80 characters or fewer"}
}
switch schedule.Kind {
case config.ReportScheduleKindResources:
case config.ReportScheduleKindBackupHealth:
for _, resource := range schedule.Scope.Resources {
kind := reporting.CanonicalResourceType(resource.ResourceType)
if kind != "vm" && kind != "system-container" {
return schedule, reportScheduleValidationError{code: "invalid_scope", message: "Backup health reports support VMs and containers"}
}
}
case config.ReportScheduleKindVMInventory:
if schedule.Format == "" {
schedule.Format = string(reporting.DescribeVMInventoryExport().Format)
}
if !reporting.DescribeVMInventoryExport().SupportsFormat(reporting.ReportFormat(schedule.Format)) {
return schedule, reportScheduleValidationError{code: "invalid_format", message: "VM inventory schedules require CSV format"}
}
if len(schedule.Scope.Resources) > 0 || len(schedule.Scope.Tags) > 0 {
return schedule, reportScheduleValidationError{code: "invalid_scope", message: "VM inventory includes all VMs in this organization. Resource and tag filters are not supported."}
}
case config.ReportScheduleKindPatrolDigest:
prepared, err := preparePatrolDigestSchedule(schedule, now)
if err != nil {
return prepared, err
}
return prepared, h.validateEnabledReportDelivery(ctx, prepared)
default:
return schedule, reportScheduleValidationError{code: "invalid_kind", message: "Schedule kind must be resources, backup_health, vm_inventory or patrol_digest"}
}
if schedule.Format == "" {
schedule.Format = config.ReportScheduleFormatPDF
}
if schedule.Format != config.ReportScheduleFormatPDF && schedule.Format != config.ReportScheduleFormatCSV {
return schedule, reportScheduleValidationError{code: "invalid_format", message: "Schedule format must be pdf or csv"}
}
if schedule.Delivery.Method == "" {
schedule.Delivery.Method = config.ReportScheduleDeliveryEmail
}
if schedule.Delivery.Method != config.ReportScheduleDeliveryEmail && schedule.Delivery.Method != config.ReportScheduleDeliveryDisk {
return schedule, reportScheduleValidationError{code: "invalid_delivery", message: "Delivery method must be email or disk"}
}
if !schedule.Delivery.Attach && !schedule.Delivery.SaveToDisk {
schedule.Delivery.Attach = true
schedule.Delivery.SaveToDisk = true
}
// Resource reports have always been retained on disk, including email runs.
schedule.Delivery.SaveToDisk = true
if err := validateReportScheduleCadence(schedule.Cadence); err != nil {
return schedule, err
}
if schedule.Kind == config.ReportScheduleKindResources {
if err := h.validateReportScheduleScope(ctx, schedule.Scope); err != nil {
return schedule, err
}
} else if schedule.Kind == config.ReportScheduleKindBackupHealth {
if err := validateReportScheduleScopeFields(schedule.Scope); err != nil {
return schedule, err
}
}
next, err := nextReportScheduleRunAt(schedule, now)
if err != nil {
return schedule, err
}
schedule.NextRunAt = &next
return schedule, h.validateEnabledReportDelivery(ctx, schedule)
}
func validateReportScheduleCadence(cadence config.ReportScheduleCadence) error {
if cadence.Type != config.ReportScheduleCadenceMonthly && cadence.Type != config.ReportScheduleCadenceWeekly {
return reportScheduleValidationError{code: "invalid_cadence", message: "Cadence must be monthly or weekly"}
}
if cadence.Type == config.ReportScheduleCadenceMonthly && (cadence.DayOfMonth < 1 || cadence.DayOfMonth > 28) {
return reportScheduleValidationError{code: "invalid_day_of_month", message: "Monthly schedules must use a day from 1 to 28"}
}
if cadence.Type == config.ReportScheduleCadenceWeekly {
if _, ok := parseReportScheduleWeekday(cadence.Weekday); !ok {
return reportScheduleValidationError{code: "invalid_weekday", message: "Weekly schedules must include a weekday"}
}
}
if _, err := time.Parse("15:04", cadence.Time); err != nil {
return reportScheduleValidationError{code: "invalid_time", message: "Schedule time must use HH:MM"}
}
if _, err := reportScheduleLocation(cadence.Timezone); err != nil {
return reportScheduleValidationError{code: "invalid_timezone", message: "Schedule timezone must be a valid IANA timezone"}
}
return nil
}
func (h *ReportingHandlers) validateReportScheduleScope(ctx context.Context, scope config.ReportScheduleScope) error {
if len(scope.Resources) == 0 && len(scope.Tags) == 0 {
return reportScheduleValidationError{code: "invalid_scope", message: "Select at least one resource or tag"}
}
definition := performanceReportDefinition()
if len(scope.Resources) > definition.MultiResourceMax {
return reportScheduleValidationError{code: "too_many_resources", message: fmt.Sprintf("Maximum %d resources allowed", definition.MultiResourceMax)}
}
if err := validateReportScheduleScopeFields(scope); err != nil {
return err
}
if len(scope.Tags) > 0 {
if _, err := h.resolveReportScheduleResources(ctx, GetOrgID(ctx), scope); err != nil {
return err
}
}
return nil
}
// Field validity is shared; scope cardinality and resolution belong to the
// report kind. Backup health intentionally supports empty and large estates.
func validateReportScheduleScopeFields(scope config.ReportScheduleScope) error {
for _, res := range scope.Resources {
if !validResourceID.MatchString(res.ResourceID) || len(res.ResourceID) > 128 {
return reportScheduleValidationError{code: "invalid_resource_id", message: "Resource IDs must match [a-zA-Z0-9._:-]+ and be 128 characters or fewer"}
}
if _, err := normalizeReportResourceType(res.ResourceType); err != nil {
return reportScheduleValidationError{code: "invalid_resource_type", message: err.Error()}
}
}
for _, tag := range scope.Tags {
if len(tag) > 64 || strings.ContainsAny(tag, "\r\n\t") {
return reportScheduleValidationError{code: "invalid_tag", message: "Tags must be 64 characters or fewer and cannot contain control characters"}
}
}
return nil
}
func (h *ReportingHandlers) resolveReportScheduleResources(ctx context.Context, orgID string, scope config.ReportScheduleScope) ([]multiReportResourceEntry, error) {
definition := performanceReportDefinition()
entries := make([]multiReportResourceEntry, 0, len(scope.Resources))
seen := map[string]struct{}{}
appendEntry := func(resourceType, resourceID string) error {
resourceType, err := normalizeReportResourceType(resourceType)
if err != nil {
return err
}
key := resourceType + "\x00" + resourceID
if _, exists := seen[key]; exists {
return nil
}
seen[key] = struct{}{}
entries = append(entries, multiReportResourceEntry{ResourceType: resourceType, ResourceID: resourceID})
return nil
}
// A schedule reports only on resources its own organization holds, so
// both explicit resources and tags resolve against that organization's
// inventory.
snapshot, ok := h.getReportingEnrichmentSnapshot(ctx, orgID)
if !ok {
return nil, reportScheduleValidationError{code: "scope_unavailable", message: "Schedule scope cannot be resolved until resource inventory is available"}
}
for _, res := range scope.Resources {
if _, err := normalizeReportResourceType(res.ResourceType); err != nil {
return nil, reportScheduleValidationError{code: "invalid_resource_type", message: err.Error()}
}
if !reportSubjectInInventory(res.ResourceID, snapshot) {
// A resource removed since the schedule was saved, or never
// part of this organization, is left out of the report.
log.Warn().
Str("org_id", orgID).
Str("resource_id", res.ResourceID).
Msg("report schedules: skipping a resource that is not in this organization's inventory")
continue
}
if err := appendEntry(res.ResourceType, res.ResourceID); err != nil {
return nil, reportScheduleValidationError{code: "invalid_resource_type", message: err.Error()}
}
}
if len(scope.Tags) > 0 {
tags := map[string]struct{}{}
for _, tag := range scope.Tags {
tags[strings.ToLower(strings.TrimSpace(tag))] = struct{}{}
}
for _, resource := range snapshot.Resources {
if !resourceHasAnyReportScheduleTag(resource, tags) {
continue
}
resourceType := reporting.CanonicalResourceType(string(resource.Type))
if resourceType == "" || strings.TrimSpace(resource.ID) == "" {
continue
}
if err := appendEntry(resourceType, resource.ID); err != nil {
continue
}
}
}
if len(entries) == 0 {
return nil, reportScheduleValidationError{code: "empty_scope", message: "Schedule scope did not match any reportable resources"}
}
if len(entries) > definition.MultiResourceMax {
return nil, reportScheduleValidationError{code: "too_many_resources", message: fmt.Sprintf("Maximum %d resources allowed", definition.MultiResourceMax)}
}
return entries, nil
}
func resourceHasAnyReportScheduleTag(resource unifiedresources.Resource, tags map[string]struct{}) bool {
for _, tag := range resource.Tags {
if _, ok := tags[strings.ToLower(strings.TrimSpace(tag))]; ok {
return true
}
}
return false
}
type reportScheduleRunResult struct {
path string
email string
usagePeriod string
}
func (h *ReportingHandlers) runReportSchedule(ctx context.Context, persistence *config.ConfigPersistence, schedule config.ReportSchedule, now time.Time, manual bool, occurrenceKey string) (reportScheduleRunResult, config.ReportSchedule) {
schedule = config.NormalizeReportSchedule(schedule)
result := reportScheduleRunResult{}
if schedule.RunAttempts < 1 {
schedule.RunAttempts = 1
}
schedule.LastRunAt = &now
schedule.UpdatedAt = now
if occurrenceKey != "" {
schedule.LastOccurrenceKey = occurrenceKey
}
if next, err := nextReportScheduleRunAt(schedule, now); err == nil {
schedule.NextRunAt = &next
}
if err := h.requireCommercialFeature(ctx, featureAdvancedReportingValue); err != nil {
return result, markReportScheduleFailed(schedule, now, fmt.Errorf("advanced reporting entitlement required: %w", err))
}
if schedule.Kind == config.ReportScheduleKindPatrolDigest {
return h.runPatrolDigestSchedule(ctx, persistence, schedule, now)
}
if schedule.Delivery.Method == config.ReportScheduleDeliveryEmail {
if _, err := scheduledReportEmailManager(persistence, schedule); err != nil {
failed := markReportScheduleDeliveryFailed(schedule, now, err)
h.recordReportingDeliveryFailure(schedule.Kind, failed.LastDeliveryStatus, now)
return result, failed
}
}
report, err := h.generateScheduledFileReport(ctx, schedule, now, manual)
if err != nil {
return result, markReportScheduleFailed(schedule, now, err)
}
h.recordReportingActivity(config.ReportingActivityEvent{At: now, Kind: schedule.Kind, Event: "generated", DataState: report.DataState})
result.usagePeriod = reportingActivityPeriod(schedule, report.Start, report.End, now)
path, err := saveGeneratedReport(persistence, schedule, report)
if err != nil {
return result, markReportScheduleFailed(schedule, now, err)
}
result.path = path
h.recordReportingActivity(config.ReportingActivityEvent{At: now, Kind: schedule.Kind, Event: "saved"})
schedule.LastReportDataState = report.DataState
schedule.LastReportDataMessage = report.DataMessage
schedule.LastReportPath = path
schedule.LastDeliveryStatus = "saved"
if schedule.Delivery.Method == config.ReportScheduleDeliveryEmail {
emailStatus, err := sendScheduledReportEmail(persistence, schedule, report, path)
if err != nil {
failed := markReportScheduleDeliveryFailed(schedule, now, err)
h.recordReportingDeliveryFailure(schedule.Kind, failed.LastDeliveryStatus, now)
return result, failed
}
result.email = emailStatus
schedule.LastDeliveryStatus = emailStatus
h.recordReportingActivity(config.ReportingActivityEvent{At: now, Kind: schedule.Kind, Event: "email_accepted"})
}
schedule.LastRunStatus = config.ReportScheduleLastRunOK
schedule.LastError = ""
schedule.RetryAt = nil
schedule.LastSuccessAt = &now
return result, schedule
}
func (h *ReportingHandlers) generateScheduledFileReport(ctx context.Context, schedule config.ReportSchedule, now time.Time, manual bool) (generatedMultiReport, error) {
if schedule.Kind == config.ReportScheduleKindBackupHealth {
return h.generateBackupHealthReport(ctx, schedule, now, manual)
}
orgID := GetOrgID(ctx)
if schedule.Kind == config.ReportScheduleKindVMInventory {
snapshot, ok := h.getReportingEnrichmentSnapshot(ctx, orgID)
if !ok {
return generatedMultiReport{}, errors.New(vmInventoryUnavailableMessage)
}
return generateVMInventoryReport(snapshot, now)
}
resources, err := h.resolveReportScheduleResources(ctx, orgID, schedule.Scope)
if err != nil {
return generatedMultiReport{}, err
}
windowEnd := now
if schedule.RunWindowEnd != nil {
windowEnd = *schedule.RunWindowEnd
}
start, end, err := reportScheduleWindow(schedule, windowEnd, manual)
if err != nil {
return generatedMultiReport{}, err
}
return h.generateMultiReportFromBody(ctx, multiReportRequestBody{
Resources: resources, Format: schedule.Format,
Start: start.UTC().Format(time.RFC3339), End: end.UTC().Format(time.RFC3339), Title: schedule.Name,
}, now)
}
func markReportScheduleFailed(schedule config.ReportSchedule, now time.Time, err error) config.ReportSchedule {
schedule.LastRunStatus = config.ReportScheduleLastRunFailed
schedule.LastError = strings.TrimSpace(err.Error())
schedule.LastRunAt = &now
schedule.UpdatedAt = now
schedule.RetryAt = nil
return schedule
}
func markReportScheduleDeliveryFailed(schedule config.ReportSchedule, now time.Time, err error) config.ReportSchedule {
schedule = markReportScheduleFailed(schedule, now, err)
schedule.LastDeliveryStatus = "failed"
var unconfirmed *notifications.EmailAcceptanceUnconfirmedError
if errors.As(err, &unconfirmed) {
schedule.LastRunStatus = config.ReportScheduleLastRunUnconfirmed
schedule.LastDeliveryStatus = "unconfirmed"
schedule.LastError = "The mail server may have accepted the report. Check the recipient before retrying to avoid a duplicate."
return schedule
}
class := notifications.ClassifyNotificationFailureError(err)
switch class {
case notifications.NotificationFailureAuthentication:
schedule.LastError = "Email authentication failed. Check the credentials in Alerts > Notifications."
case notifications.NotificationFailureConfiguration:
var setup reportScheduleValidationError
if errors.As(err, &setup) {
schedule.LastError = setup.message
} else {
schedule.LastError = "Email configuration was rejected. Check Alerts > Notifications."
}
case notifications.NotificationFailureRejected:
schedule.LastError = "The mail server rejected this report. Check the recipient and notification settings."
case notifications.NotificationFailureTLS:
schedule.LastError = "The secure mail connection failed. Check the server certificate and TLS settings in Alerts > Notifications."
default:
schedule.LastError = "The mail server is temporarily unavailable. Check the connection and notification settings."
}
if class.Retryable() && schedule.Enabled && schedule.RunAttempts < reportScheduleMaxAttempts {
delay := time.Minute
if schedule.RunAttempts >= 2 {
delay = 5 * time.Minute
}
retryAt := now.Add(delay)
schedule.RetryAt = &retryAt
schedule.LastRunStatus = config.ReportScheduleLastRunRetrying
} else if class.Retryable() && schedule.RunAttempts >= reportScheduleMaxAttempts {
schedule.LastError = "Delivery failed after 3 attempts. " + schedule.LastError
}
return schedule
}
func reportScheduleWindow(schedule config.ReportSchedule, now time.Time, manual bool) (time.Time, time.Time, error) {
location, err := reportScheduleLocation(schedule.Cadence.Timezone)
if err != nil {
return time.Time{}, time.Time{}, err
}
end := now.In(location)
if !manual {
occurrence, _, err := lastReportScheduleOccurrenceAt(schedule, now)
if err == nil {
end = occurrence.In(location)
}
}
switch schedule.Cadence.Type {
case config.ReportScheduleCadenceMonthly:
firstThisMonth := time.Date(end.Year(), end.Month(), 1, 0, 0, 0, 0, location)
firstPreviousMonth := firstThisMonth.AddDate(0, -1, 0)
return firstPreviousMonth.UTC(), firstThisMonth.UTC(), nil
case config.ReportScheduleCadenceWeekly:
return end.AddDate(0, 0, -7).UTC(), end.UTC(), nil
default:
return time.Time{}, time.Time{}, fmt.Errorf("unsupported cadence %q", schedule.Cadence.Type)
}
}
func saveGeneratedReport(persistence *config.ConfigPersistence, schedule config.ReportSchedule, report generatedMultiReport) (string, error) {
baseDir, err := securityutil.JoinStorageLeaf(persistence.DataDir(), "reports")
if err != nil {
return "", err
}
generatedDir, err := securityutil.JoinStorageLeaf(baseDir, "generated")
if err != nil {
return "", err
}
scheduleDir, err := securityutil.JoinStorageLeaf(generatedDir, securityutil.HashedStorageName(schedule.ID))
if err != nil {
return "", err
}
if err := os.MkdirAll(scheduleDir, 0o700); err != nil {
return "", fmt.Errorf("create report output directory: %w", err)
}
filename := sanitizeFilename(report.Filename)
if filename == "" {
filename = "report." + string(report.Format)
}
path, err := securityutil.JoinStorageLeaf(scheduleDir, filename)
if err != nil {
return "", err
}
if err := os.WriteFile(path, report.Data, 0o600); err != nil {
return "", fmt.Errorf("write generated report: %w", err)
}
pruneGeneratedReports(scheduleDir, schedule.RetentionCount)
return path, nil
}
func pruneGeneratedReports(dir string, retentionCount int) {
if retentionCount <= 0 {
retentionCount = config.DefaultReportScheduleRetentionCount
}
entries, err := os.ReadDir(dir)
if err != nil {
return
}
type generatedFile struct {
path string
modTime time.Time
}
files := make([]generatedFile, 0, len(entries))
for _, entry := range entries {
if entry.IsDir() {
continue
}
info, err := entry.Info()
if err != nil {
continue
}
path, err := securityutil.JoinStorageLeaf(dir, entry.Name())
if err != nil {
continue
}
files = append(files, generatedFile{path: path, modTime: info.ModTime()})
}
sort.Slice(files, func(i, j int) bool {
return files[i].modTime.After(files[j].modTime)
})
for i := retentionCount; i < len(files); i++ {
_ = os.Remove(files[i].path)
}
}
func reportEmailConfigurationFailure(message string) error {
return notifications.FailWithClass(notifications.NotificationFailureConfiguration,
reportScheduleValidationError{code: "email_not_ready", message: message})
}
// Configuration validation and SMTP semantics remain owned by notifications.
// Reporting persists the retry budget, so one attempt must be one SMTP send.
func scheduledReportEmailManager(persistence *config.ConfigPersistence, schedule config.ReportSchedule) (*notifications.EnhancedEmailManager, error) {
if persistence == nil {
return nil, reportEmailConfigurationFailure("Email configuration is unavailable. Check Alerts > Notifications.")
}
emailCfg, err := persistence.LoadEmailConfig()
if err != nil {
return nil, reportEmailConfigurationFailure("Email configuration could not be loaded. Check notification settings and server storage.")
}
if emailCfg == nil || !emailCfg.Enabled {
return nil, reportEmailConfigurationFailure("Enable email notifications in Alerts > Notifications before running this report.")
}
recipients := schedule.Delivery.To
if len(recipients) == 0 {
recipients = emailCfg.To
}
if len(recipients) == 0 && strings.TrimSpace(emailCfg.From) != "" {
recipients = []string{emailCfg.From}
}
if len(recipients) == 0 {
return nil, reportEmailConfigurationFailure("Add report recipients or email notification recipients in Alerts > Notifications.")
}
providerConfig := notifications.EmailProviderConfig{
EmailConfig: notifications.EmailConfig{
Provider: emailCfg.Provider,
From: emailCfg.From,
To: recipients,
SMTPHost: emailCfg.SMTPHost,
SMTPPort: emailCfg.SMTPPort,
Username: emailCfg.Username,
Password: emailCfg.Password,
TLS: emailCfg.TLS,
StartTLS: emailCfg.StartTLS,
RateLimit: emailCfg.RateLimit,
},
Provider: emailCfg.Provider,
MaxRetries: 0,
RateLimit: emailCfg.RateLimit,
StartTLS: emailCfg.StartTLS,
AuthRequired: emailCfg.Username != "" && emailCfg.Password != "",
}
manager := notifications.NewEnhancedEmailManager(providerConfig)
if err := manager.ValidateConfiguration(); err != nil {
return nil, reportEmailConfigurationFailure(err.Error())
}
return manager, nil
}
func sendScheduledReportEmail(persistence *config.ConfigPersistence, schedule config.ReportSchedule, report generatedMultiReport, path string) (string, error) {
manager, err := scheduledReportEmailManager(persistence, schedule)
if err != nil {
return "", err
}
subject := "Pulse report: " + schedule.Name
htmlBody := "<p>Your scheduled Pulse report is ready.</p>"
textBody := "Your scheduled Pulse report is ready."
attachments := []notifications.EmailAttachment{}
if schedule.Delivery.Attach && len(report.Data) <= reportScheduleAttachmentLimitBytes {
attachments = append(attachments, notifications.EmailAttachment{
Filename: report.Filename,
ContentType: report.ContentType,
Data: report.Data,
})
} else if schedule.Delivery.Attach && len(report.Data) > reportScheduleAttachmentLimitBytes {
return "", reportEmailConfigurationFailure("The report exceeds the 15 MiB attachment limit. Choose CSV or turn off Attach report to email a saved-path notice.")
} else {
htmlBody = "<p>Your scheduled Pulse report was generated and saved to disk.</p><p>Saved path: " + htmlEscape(path) + "</p>"
textBody = "Your scheduled Pulse report was generated and saved to disk.\nSaved path: " + path
}
if err := manager.SendEmailWithAttachments(subject, htmlBody, textBody, attachments); err != nil {
return "", err
}
if len(attachments) > 0 {
return "sent_with_attachment", nil
}
return "sent_saved_path", nil
}
func htmlEscape(value string) string {
replacer := strings.NewReplacer("&", "&", "<", "<", ">", ">", "\"", """, "'", "'")
return replacer.Replace(value)
}
func reportScheduleLocation(name string) (*time.Location, error) {
name = strings.TrimSpace(name)
if name == "" || name == "Local" {
name = "UTC"
}
return time.LoadLocation(name)
}
func parseReportScheduleClock(clock string) (hour, minute int, err error) {
parsed, err := time.Parse("15:04", strings.TrimSpace(clock))
if err != nil {
return 0, 0, err
}
return parsed.Hour(), parsed.Minute(), nil
}
func parseReportScheduleWeekday(value string) (time.Weekday, bool) {
switch strings.ToLower(strings.TrimSpace(value)) {
case "sunday", "sun":
return time.Sunday, true
case "monday", "mon":
return time.Monday, true
case "tuesday", "tue":
return time.Tuesday, true
case "wednesday", "wed":
return time.Wednesday, true
case "thursday", "thu":
return time.Thursday, true
case "friday", "fri":
return time.Friday, true
case "saturday", "sat":
return time.Saturday, true
default:
return time.Sunday, false
}
}
func nextReportScheduleRunAt(schedule config.ReportSchedule, after time.Time) (time.Time, error) {
location, err := reportScheduleLocation(schedule.Cadence.Timezone)
if err != nil {
return time.Time{}, err
}
hour, minute, err := parseReportScheduleClock(schedule.Cadence.Time)
if err != nil {
return time.Time{}, err
}
localAfter := after.In(location)
switch schedule.Cadence.Type {
case config.ReportScheduleCadenceMonthly:
candidate := time.Date(localAfter.Year(), localAfter.Month(), schedule.Cadence.DayOfMonth, hour, minute, 0, 0, location)
if !candidate.After(localAfter) {
candidate = candidate.AddDate(0, 1, 0)
}
return candidate.UTC(), nil
case config.ReportScheduleCadenceWeekly:
weekday, ok := parseReportScheduleWeekday(schedule.Cadence.Weekday)
if !ok {
return time.Time{}, fmt.Errorf("invalid weekday")
}
dayDelta := (int(weekday) - int(localAfter.Weekday()) + 7) % 7
candidateDate := localAfter.AddDate(0, 0, dayDelta)
candidate := time.Date(candidateDate.Year(), candidateDate.Month(), candidateDate.Day(), hour, minute, 0, 0, location)
if !candidate.After(localAfter) {
candidate = candidate.AddDate(0, 0, 7)
}
return candidate.UTC(), nil
default:
return time.Time{}, fmt.Errorf("unsupported cadence %q", schedule.Cadence.Type)
}
}
func lastReportScheduleOccurrenceAt(schedule config.ReportSchedule, now time.Time) (time.Time, string, error) {
location, err := reportScheduleLocation(schedule.Cadence.Timezone)
if err != nil {
return time.Time{}, "", err
}
hour, minute, err := parseReportScheduleClock(schedule.Cadence.Time)
if err != nil {
return time.Time{}, "", err
}
localNow := now.In(location)
var occurrence time.Time
switch schedule.Cadence.Type {
case config.ReportScheduleCadenceMonthly:
occurrence = time.Date(localNow.Year(), localNow.Month(), schedule.Cadence.DayOfMonth, hour, minute, 0, 0, location)
if occurrence.After(localNow) {
occurrence = occurrence.AddDate(0, -1, 0)
}
case config.ReportScheduleCadenceWeekly:
weekday, ok := parseReportScheduleWeekday(schedule.Cadence.Weekday)
if !ok {
return time.Time{}, "", fmt.Errorf("invalid weekday")
}
dayDelta := (int(localNow.Weekday()) - int(weekday) + 7) % 7
occurrenceDate := localNow.AddDate(0, 0, -dayDelta)
occurrence = time.Date(occurrenceDate.Year(), occurrenceDate.Month(), occurrenceDate.Day(), hour, minute, 0, 0, location)
if occurrence.After(localNow) {
occurrence = occurrence.AddDate(0, 0, -7)
}
default:
return time.Time{}, "", fmt.Errorf("unsupported cadence %q", schedule.Cadence.Type)
}
return occurrence.UTC(), occurrenceKey(schedule, occurrence), nil
}
func occurrenceKey(schedule config.ReportSchedule, occurrence time.Time) string {
timezone := strings.TrimSpace(schedule.Cadence.Timezone)
if timezone == "" {
timezone = "UTC"
}
return schedule.Cadence.Type + ":" + occurrence.UTC().Format("2006-01-02T15:04:05Z07:00") + ":" + timezone
}
func (h *ReportingHandlers) RunReportScheduleScheduler(ctx context.Context) {
if h == nil {
return
}
h.runDueReportSchedules(ctx, time.Now().UTC())
ticker := time.NewTicker(reportScheduleTickInterval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case now := <-ticker.C:
h.runDueReportSchedules(ctx, now.UTC())
}
}
}
func (h *ReportingHandlers) runDueReportSchedules(ctx context.Context, now time.Time) {
manager := h.reportMonitorManager()
if manager == nil {
return
}
orgIDs, err := manager.ListOrganizationIDs()
if err != nil {
log.Warn().Err(err).Msg("report schedules: list organizations")
return
}
for _, orgID := range orgIDs {
orgCtx := context.WithValue(ctx, OrgIDContextKey, orgID)
h.syncCommercialHistoryRetention(orgCtx, orgID, now)
h.runDueReportSchedulesForOrg(orgCtx, orgID, now)
}
}
func (h *ReportingHandlers) syncCommercialHistoryRetention(ctx context.Context, orgID string, now time.Time) {
manager := h.reportMonitorManager()
if manager == nil || h.commercialLicenseResolver == nil {
return
}
monitor, err := manager.GetMonitor(orgID)
if err != nil || monitor == nil || monitor.GetMetricsStore() == nil {
return
}
service := h.commercialLicenseResolver(ctx)
if service == nil {
monitor.GetMetricsStore().SetCommercialHistoryRetention(0, time.Time{})
return
}
activation := service.GetActivationState()
if activation == nil || activation.Continuity.DowngradeRetention == nil {
monitor.GetMetricsStore().SetCommercialHistoryRetention(0, time.Time{})
return
}
downgrade := normalizeDowngradeRetentionStateFromLicensing(*activation.Continuity.DowngradeRetention)
monitor.GetMetricsStore().SetCommercialHistoryRetention(
downgrade.CurrentHistoryDays,
time.Unix(downgrade.PurgeEligibleAt, 0).UTC(),
)
if now.UTC().Unix() < downgrade.PurgeEligibleAt || service.RequireFeature(featureAdvancedReportingValue) == nil {
return
}
persistence, err := h.reportSchedulePersistence(ctx)
if err != nil {
return
}
if err := purgeGeneratedReportsAfterDowngrade(persistence); err != nil {
log.Warn().Err(err).Str("org_id", orgID).Msg("report schedules: purge downgrade artifacts")
}
}
func purgeGeneratedReportsAfterDowngrade(persistence *config.ConfigPersistence) error {
if persistence == nil {
return nil
}
baseDir, err := securityutil.JoinStorageLeaf(persistence.DataDir(), "reports")
if err != nil {
return err
}
generatedDir, err := securityutil.JoinStorageLeaf(baseDir, "generated")
if err != nil {
return err
}
scheduleDirs, err := os.ReadDir(generatedDir)
if errors.Is(err, os.ErrNotExist) {
return nil
}
if err != nil {
return err
}
for _, scheduleDir := range scheduleDirs {
if !scheduleDir.IsDir() || scheduleDir.Type()&os.ModeSymlink != 0 {
continue
}
dir, err := securityutil.JoinStorageLeaf(generatedDir, scheduleDir.Name())
if err != nil {
continue
}
files, err := os.ReadDir(dir)
if err != nil {
continue
}
for _, file := range files {
if file.IsDir() || file.Type()&os.ModeSymlink != 0 {
continue
}
path, err := securityutil.JoinStorageLeaf(dir, file.Name())
if err == nil {
_ = os.Remove(path)
}
}
}
return nil
}
func (h *ReportingHandlers) runDueReportSchedulesForOrg(ctx context.Context, orgID string, now time.Time) {
h.scheduleRunMu.Lock()
defer h.scheduleRunMu.Unlock()
persistence, store, err := h.reportSchedulePersistenceAndStore(ctx)
if err != nil {
log.Warn().Err(err).Str("org_id", orgID).Msg("report schedules: load store")
return
}
save := func(index int, schedule config.ReportSchedule) bool {
store.Schedules[index] = schedule
if err := persistence.SaveReportScheduleStore(*store); err != nil {
log.Warn().Err(err).Str("org_id", orgID).Msg("report schedules: save status")
return false
}
return true
}
for i := range store.Schedules {
if ctx.Err() != nil {
return
}
schedule := config.NormalizeReportSchedule(store.Schedules[i])
if schedule.LastRunStatus == config.ReportScheduleLastRunRunning {
schedule.UpdatedAt = now
schedule.RetryAt = nil
schedule.LastRunStatus = config.ReportScheduleLastRunFailed
schedule.LastError = "The previous report run ended before completion was recorded. Run it again."
if schedule.Delivery.Method == config.ReportScheduleDeliveryEmail {
schedule.LastRunStatus = config.ReportScheduleLastRunUnconfirmed
schedule.LastDeliveryStatus = "unconfirmed"
schedule.LastError = "The previous email run ended before delivery was recorded. Check the recipient before retrying to avoid a duplicate."
} else if schedule.Enabled && schedule.RunWindowEnd != nil && schedule.RunAttempts < reportScheduleMaxAttempts {
schedule.LastRunStatus = config.ReportScheduleLastRunRetrying
schedule.LastError = "Report generation was interrupted. Retrying the saved report window."
schedule.RetryAt = &now
}
if !save(i, schedule) {
return
}
if schedule.LastRunStatus == config.ReportScheduleLastRunUnconfirmed {
h.recordReportingDeliveryFailure(schedule.Kind, schedule.LastDeliveryStatus, now)
}
}
if !schedule.Enabled {
if schedule.RetryAt != nil {
schedule.RetryAt = nil
schedule.RunWindowEnd = nil
schedule.LastRunStatus = config.ReportScheduleLastRunFailed
schedule.LastError = "Automatic retry cancelled because this schedule is disabled."
schedule.UpdatedAt = now
if !save(i, schedule) {
return
}
}
continue
}
if schedule.RetryAt != nil {
if schedule.LastRunStatus != config.ReportScheduleLastRunRetrying || schedule.RunWindowEnd == nil || schedule.RunAttempts >= reportScheduleMaxAttempts {
schedule.RetryAt = nil
schedule.LastRunStatus = config.ReportScheduleLastRunFailed
schedule.LastError = "The saved retry is incomplete or exhausted. Run a new report."
if !save(i, schedule) {
return
}
continue
}
if now.Before(*schedule.RetryAt) {
continue
}
if _, _, err := h.executeReportSchedule(ctx, persistence, store, i, now, false, "", true); err != nil {
log.Warn().Err(err).Str("org_id", orgID).Msg("report schedules: persist retry outcome")
return
}
continue
}
occurrence, key, err := lastReportScheduleOccurrenceAt(schedule, now)
if err != nil {
if !save(i, markReportScheduleFailed(schedule, now, err)) {
return
}
continue
}
if key == "" || key == schedule.LastOccurrenceKey || occurrence.After(now) {
continue
}
if now.Sub(occurrence) > reportScheduleMissedRunGrace {
schedule.LastOccurrenceKey = key
if next, err := nextReportScheduleRunAt(schedule, now); err == nil {
schedule.NextRunAt = &next
}
schedule.UpdatedAt = now
if !save(i, schedule) {
return
}
continue
}
if _, _, err := h.executeReportSchedule(ctx, persistence, store, i, now, false, key, false); err != nil {
log.Warn().Err(err).Str("org_id", orgID).Msg("report schedules: persist outcome")
return
}
}
}
// preparePatrolDigestSchedule validates the Patrol weekly summary kind. The
// summary is fleet-wide and only makes sense as a weekly email, so scope,
// format, attachments, and disk delivery are fixed rather than user choices.
func preparePatrolDigestSchedule(schedule config.ReportSchedule, now time.Time) (config.ReportSchedule, error) {
schedule.Format = config.ReportScheduleFormatEmail
schedule.Scope = config.ReportScheduleScope{Resources: []config.ReportScheduleResource{}, Tags: []string{}}
if schedule.Delivery.Method == "" {
schedule.Delivery.Method = config.ReportScheduleDeliveryEmail
}
if schedule.Delivery.Method != config.ReportScheduleDeliveryEmail {
return schedule, reportScheduleValidationError{code: "invalid_delivery", message: "Patrol weekly summaries are delivered by email"}
}
schedule.Delivery.Attach = false
schedule.Delivery.SaveToDisk = false
if schedule.Cadence.Type != config.ReportScheduleCadenceWeekly {
return schedule, reportScheduleValidationError{code: "invalid_cadence", message: "Patrol weekly summaries use a weekly cadence"}
}
if err := validateReportScheduleCadence(schedule.Cadence); err != nil {
return schedule, err
}
next, err := nextReportScheduleRunAt(schedule, now)
if err != nil {
return schedule, err
}
schedule.NextRunAt = &next
return schedule, nil
}
// runPatrolDigestSchedule emails the last seven days of Patrol work. Nothing is
// written to disk: the digest is a rollup of records Pulse already keeps, so
// the email is the only artifact.
func (h *ReportingHandlers) runPatrolDigestSchedule(ctx context.Context, persistence *config.ConfigPersistence, schedule config.ReportSchedule, now time.Time) (reportScheduleRunResult, config.ReportSchedule) {
result := reportScheduleRunResult{}
if h == nil || h.patrolDigestResolver == nil {
return result, markReportScheduleFailed(schedule, now, errors.New("Patrol weekly summary is unavailable on this server"))
}
digest, ok := h.patrolDigestResolver(ctx, ai.PatrolDigestDefaultDays)
if !ok {
return result, markReportScheduleFailed(schedule, now, errors.New("Patrol is not available for this workspace"))
}
manager, err := scheduledReportEmailManager(persistence, schedule)
if err != nil {
failed := markReportScheduleDeliveryFailed(schedule, now, err)
h.recordReportingDeliveryFailure(schedule.Kind, failed.LastDeliveryStatus, now)
return result, failed
}
subject, htmlBody, textBody := renderPatrolDigestEmail(digest, h.patrolDigestPublicURL())
h.recordReportingActivity(config.ReportingActivityEvent{At: now, Kind: schedule.Kind, Event: "generated"})
result.usagePeriod = reportingActivityPeriod(schedule, time.Time{}, time.Time{}, now)
if err := manager.SendEmailWithRetry(subject, htmlBody, textBody); err != nil {
failed := markReportScheduleDeliveryFailed(schedule, now, err)
h.recordReportingDeliveryFailure(schedule.Kind, failed.LastDeliveryStatus, now)
return result, failed
}
h.recordReportingActivity(config.ReportingActivityEvent{At: now, Kind: schedule.Kind, Event: "email_accepted"})
result.email = "sent"
schedule.LastRunStatus = config.ReportScheduleLastRunOK
schedule.LastError = ""
schedule.LastDeliveryStatus = result.email
schedule.RetryAt = nil
schedule.LastSuccessAt = &now
return result, schedule
}
func (h *ReportingHandlers) patrolDigestPublicURL() string {
if h == nil || h.settingsStore == nil {
return ""
}
settings, err := h.settingsStore.LoadSystemSettings()
if err != nil || settings == nil {
return ""
}
return strings.TrimRight(strings.TrimSpace(settings.PublicURL), "/")
}