Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

51.8 KB
Raw Blame
package api

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"os"
	"sort"
	"strings"
	"time"

	"github.com/google/uuid"
	"github.com/rcourtman/pulse-go-rewrite/internal/ai"
	"github.com/rcourtman/pulse-go-rewrite/internal/config"
	"github.com/rcourtman/pulse-go-rewrite/internal/notifications"
	"github.com/rcourtman/pulse-go-rewrite/internal/securityutil"
	"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
	"github.com/rcourtman/pulse-go-rewrite/pkg/reporting"
	"github.com/rs/zerolog/log"
)

const (
	reportScheduleAttachmentLimitBytes = 15 * 1024 * 1024
	reportScheduleTickInterval         = time.Minute
	reportScheduleMissedRunGrace       = 24 * time.Hour
	reportScheduleMaxAttempts          = 3
)

type reportScheduleListResponse struct {
	Schedules      []config.ReportSchedule                `json:"schedules"`
	DeliveryChecks map[string]reportScheduleDeliveryCheck `json:"delivery_checks"`
}

type reportScheduleDeliveryCheck struct {
	Ready   bool   `json:"ready"`
	Message string `json:"message"`
}

type reportScheduleRunResponse struct {
	Schedule config.ReportSchedule `json:"schedule"`
	Status   string                `json:"status"`
	Path     string                `json:"path,omitempty"`
	Email    string                `json:"email,omitempty"`
}

func (h *ReportingHandlers) HandleListReportSchedules(w http.ResponseWriter, r *http.Request) {
	if r.Method != http.MethodGet {
		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
		return
	}
	persistence, store, err := h.reportSchedulePersistenceAndStore(r.Context())
	if err != nil {
		writeErrorResponse(w, http.StatusInternalServerError, "schedule_store_unavailable", "Report schedules are unavailable", nil)
		return
	}
	checks := make(map[string]reportScheduleDeliveryCheck, len(store.Schedules))
	for _, schedule := range store.Schedules {
		checks[schedule.ID] = checkReportScheduleDelivery(persistence, schedule)
	}
	writeReportScheduleJSON(w, http.StatusOK, reportScheduleListResponse{Schedules: store.Schedules, DeliveryChecks: checks})
}

func (h *ReportingHandlers) HandleCreateReportSchedule(w http.ResponseWriter, r *http.Request) {
	if r.Method != http.MethodPost {
		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
		return
	}
	var incoming config.ReportSchedule
	if err := decodeReportScheduleBody(w, r, &incoming); err != nil {
		return
	}
	h.scheduleRunMu.Lock()
	defer h.scheduleRunMu.Unlock()

	persistence, store, err := h.reportSchedulePersistenceAndStore(r.Context())
	if err != nil {
		writeErrorResponse(w, http.StatusInternalServerError, "schedule_store_unavailable", "Report schedules are unavailable", nil)
		return
	}
	now := time.Now().UTC()
	incoming.ID = uuid.NewString()
	incoming.CreatedAt = now
	incoming.UpdatedAt = now
	copyReportScheduleRunState(&incoming, config.ReportSchedule{})
	schedule, err := h.prepareReportSchedule(r.Context(), incoming, now, false)
	if err != nil {
		writeReportScheduleValidationError(w, err)
		return
	}
	schedule.LastOccurrenceKey = currentReportScheduleOccurrenceKey(schedule, now)
	store.Schedules = append(store.Schedules, schedule)
	if err := persistence.SaveReportScheduleStore(*store); err != nil {
		writeErrorResponse(w, http.StatusInternalServerError, "schedule_save_failed", "Failed to save report schedule", nil)
		return
	}
	writeReportScheduleJSON(w, http.StatusCreated, schedule)
}

func (h *ReportingHandlers) HandleUpdateReportSchedule(w http.ResponseWriter, r *http.Request) {
	if r.Method != http.MethodPut {
		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
		return
	}
	scheduleID := strings.TrimSpace(r.PathValue("id"))
	if scheduleID == "" {
		writeErrorResponse(w, http.StatusBadRequest, "missing_schedule_id", "Schedule ID is required", nil)
		return
	}
	var incoming config.ReportSchedule
	if err := decodeReportScheduleBody(w, r, &incoming); err != nil {
		return
	}
	h.scheduleRunMu.Lock()
	defer h.scheduleRunMu.Unlock()

	persistence, store, err := h.reportSchedulePersistenceAndStore(r.Context())
	if err != nil {
		writeErrorResponse(w, http.StatusInternalServerError, "schedule_store_unavailable", "Report schedules are unavailable", nil)
		return
	}
	index := findReportScheduleIndex(store.Schedules, scheduleID)
	if index < 0 {
		writeErrorResponse(w, http.StatusNotFound, "schedule_not_found", "Report schedule not found", nil)
		return
	}

	now := time.Now().UTC()
	existing := store.Schedules[index]
	incoming.ID = existing.ID
	incoming.CreatedAt = existing.CreatedAt
	incoming.UpdatedAt = now
	copyReportScheduleRunState(&incoming, existing)
	schedule, err := h.prepareReportSchedule(r.Context(), incoming, now, false)
	if err != nil {
		writeReportScheduleValidationError(w, err)
		return
	}
	if schedule.RetryAt != nil {
		schedule.LastRunStatus = config.ReportScheduleLastRunFailed
		schedule.LastError = "Automatic retry cancelled because the schedule was edited. Run a new report when ready."
	}
	schedule.RetryAt = nil
	schedule.RunWindowEnd = nil
	schedule.RunManual = false
	schedule.LastOccurrenceKey = currentReportScheduleOccurrenceKey(schedule, now)
	store.Schedules[index] = schedule
	if err := persistence.SaveReportScheduleStore(*store); err != nil {
		writeErrorResponse(w, http.StatusInternalServerError, "schedule_save_failed", "Failed to save report schedule", nil)
		return
	}
	writeReportScheduleJSON(w, http.StatusOK, schedule)
}

func (h *ReportingHandlers) HandleDeleteReportSchedule(w http.ResponseWriter, r *http.Request) {
	if r.Method != http.MethodDelete {
		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
		return
	}
	scheduleID := strings.TrimSpace(r.PathValue("id"))
	if scheduleID == "" {
		writeErrorResponse(w, http.StatusBadRequest, "missing_schedule_id", "Schedule ID is required", nil)
		return
	}
	h.scheduleRunMu.Lock()
	defer h.scheduleRunMu.Unlock()
	persistence, store, err := h.reportSchedulePersistenceAndStore(r.Context())
	if err != nil {
		writeErrorResponse(w, http.StatusInternalServerError, "schedule_store_unavailable", "Report schedules are unavailable", nil)
		return
	}
	index := findReportScheduleIndex(store.Schedules, scheduleID)
	if index < 0 {
		writeErrorResponse(w, http.StatusNotFound, "schedule_not_found", "Report schedule not found", nil)
		return
	}
	store.Schedules = append(store.Schedules[:index], store.Schedules[index+1:]...)
	if err := persistence.SaveReportScheduleStore(*store); err != nil {
		writeErrorResponse(w, http.StatusInternalServerError, "schedule_save_failed", "Failed to save report schedule", nil)
		return
	}
	w.WriteHeader(http.StatusNoContent)
}

func (h *ReportingHandlers) HandleRunReportSchedule(w http.ResponseWriter, r *http.Request) {
	if r.Method != http.MethodPost {
		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
		return
	}
	scheduleID := strings.TrimSpace(r.PathValue("id"))
	if scheduleID == "" {
		writeErrorResponse(w, http.StatusBadRequest, "missing_schedule_id", "Schedule ID is required", nil)
		return
	}
	h.scheduleRunMu.Lock()
	defer h.scheduleRunMu.Unlock()
	persistence, store, err := h.reportSchedulePersistenceAndStore(r.Context())
	if err != nil {
		writeErrorResponse(w, http.StatusInternalServerError, "schedule_store_unavailable", "Report schedules are unavailable", nil)
		return
	}
	index := findReportScheduleIndex(store.Schedules, scheduleID)
	if index < 0 {
		writeErrorResponse(w, http.StatusNotFound, "schedule_not_found", "Report schedule not found", nil)
		return
	}

	schedule := store.Schedules[index]
	if schedule.LastRunStatus == config.ReportScheduleLastRunRunning {
		writeErrorResponse(w, http.StatusConflict, "schedule_running", "This report is already running. Refresh its status before trying again.", nil)
		return
	}
	retry := r.URL.Query().Get("retry") == "true"
	if retry && (schedule.RunWindowEnd == nil || !reportScheduleRetryableOutcome(schedule.LastRunStatus)) {
		writeErrorResponse(w, http.StatusConflict, "retry_unavailable", "The original failed report window is unavailable. Run a new report.", nil)
		return
	}
	result, updated, err := h.executeReportSchedule(r.Context(), persistence, store, index, time.Now().UTC(), true, "", retry)
	if err != nil {
		writeErrorResponse(w, http.StatusInternalServerError, "schedule_save_failed", "The report outcome could not be saved. Check its status and the recipient before running again.", nil)
		return
	}
	status := http.StatusOK
	if updated.LastRunStatus == config.ReportScheduleLastRunFailed {
		status = http.StatusInternalServerError
	} else if updated.LastRunStatus == config.ReportScheduleLastRunRetrying {
		status = http.StatusAccepted
	} else if updated.LastRunStatus == config.ReportScheduleLastRunUnconfirmed {
		status = http.StatusBadGateway
	}
	writeReportScheduleJSON(w, status, reportScheduleRunResponse{
		Schedule: updated,
		Status:   updated.LastRunStatus,
		Path:     result.path,
		Email:    result.email,
	})
}

func decodeReportScheduleBody(w http.ResponseWriter, r *http.Request, target *config.ReportSchedule) error {
	r.Body = http.MaxBytesReader(w, r.Body, reportingMultiReportBodyMax)
	decoder := json.NewDecoder(r.Body)
	decoder.DisallowUnknownFields()
	if err := decoder.Decode(target); err != nil {
		writeErrorResponse(w, http.StatusBadRequest, "invalid_body", "Invalid report schedule body", nil)
		return err
	}
	if err := decoder.Decode(&struct{}{}); err != nil && !errors.Is(err, io.EOF) {
		writeErrorResponse(w, http.StatusBadRequest, "invalid_body", "Invalid report schedule body", nil)
		return err
	}
	return nil
}

func writeReportScheduleJSON(w http.ResponseWriter, status int, payload any) {
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(status)
	_ = json.NewEncoder(w).Encode(payload)
}

type reportScheduleValidationError struct {
	code    string
	message string
}

func (e reportScheduleValidationError) Error() string {
	return e.message
}

func writeReportScheduleValidationError(w http.ResponseWriter, err error) {
	var validationErr reportScheduleValidationError
	if errors.As(err, &validationErr) {
		writeErrorResponse(w, http.StatusBadRequest, validationErr.code, validationErr.message, nil)
		return
	}
	writeErrorResponse(w, http.StatusBadRequest, "invalid_schedule", err.Error(), nil)
}

func (h *ReportingHandlers) reportSchedulePersistenceAndStore(ctx context.Context) (*config.ConfigPersistence, *config.ReportScheduleStore, error) {
	persistence, err := h.reportSchedulePersistence(ctx)
	if err != nil {
		return nil, nil, err
	}
	store, err := persistence.LoadReportScheduleStore()
	if err != nil {
		return nil, nil, err
	}
	if store == nil {
		empty := config.EmptyReportScheduleStore()
		store = &empty
	}
	return persistence, store, nil
}

func (h *ReportingHandlers) reportSchedulePersistence(ctx context.Context) (*config.ConfigPersistence, error) {
	manager := h.reportMonitorManager()
	if manager == nil {
		return nil, fmt.Errorf("multi-tenant monitor is not configured")
	}
	orgID := GetOrgID(ctx)
	if strings.TrimSpace(orgID) == "" {
		orgID = "default"
	}
	monitor, err := manager.GetMonitor(orgID)
	if err != nil {
		return nil, err
	}
	persistence := monitor.GetConfigPersistence()
	if persistence == nil {
		return nil, fmt.Errorf("config persistence is not configured")
	}
	return persistence, nil
}

func findReportScheduleIndex(schedules []config.ReportSchedule, id string) int {
	for i := range schedules {
		if schedules[i].ID == id {
			return i
		}
	}
	return -1
}

// Outcomes are server-owned even when older clients PUT a whole schedule.
func copyReportScheduleRunState(dst *config.ReportSchedule, src config.ReportSchedule) {
	dst.LastRunAt = src.LastRunAt
	dst.LastRunStatus = src.LastRunStatus
	dst.LastError = src.LastError
	dst.LastDeliveryStatus = src.LastDeliveryStatus
	dst.LastReportPath = src.LastReportPath
	dst.LastReportDataState = src.LastReportDataState
	dst.LastReportDataMessage = src.LastReportDataMessage
	dst.LastSuccessAt = src.LastSuccessAt
	dst.RunAttempts = src.RunAttempts
	dst.RetryAt = src.RetryAt
	dst.RunWindowEnd = src.RunWindowEnd
	dst.RunManual = src.RunManual
	dst.NextRunAt = src.NextRunAt
	dst.LastOccurrenceKey = src.LastOccurrenceKey
}

func currentReportScheduleOccurrenceKey(schedule config.ReportSchedule, now time.Time) string {
	_, key, _ := lastReportScheduleOccurrenceAt(schedule, now)
	return key
}

func checkReportScheduleDelivery(persistence *config.ConfigPersistence, schedule config.ReportSchedule) reportScheduleDeliveryCheck {
	if schedule.Delivery.Method == config.ReportScheduleDeliveryDisk {
		return reportScheduleDeliveryCheck{Ready: true, Message: "Reports are saved on this Pulse server."}
	}
	_, err := scheduledReportEmailManager(persistence, schedule)
	if err != nil {
		return reportScheduleDeliveryCheck{Message: err.Error()}
	}
	return reportScheduleDeliveryCheck{Ready: true, Message: "Email configuration checked. Run the report to test delivery."}
}

func (h *ReportingHandlers) validateEnabledReportDelivery(ctx context.Context, schedule config.ReportSchedule) error {
	if !schedule.Enabled || schedule.Delivery.Method != config.ReportScheduleDeliveryEmail {
		return nil
	}
	persistence, err := h.reportSchedulePersistence(ctx)
	if err != nil {
		return reportScheduleValidationError{code: "email_configuration_unavailable", message: "Email configuration is unavailable. Check Alerts > Notifications."}
	}
	check := checkReportScheduleDelivery(persistence, schedule)
	if !check.Ready {
		return reportScheduleValidationError{code: "email_not_ready", message: check.Message}
	}
	return nil
}

func reportScheduleRetryableOutcome(status string) bool {
	return status == config.ReportScheduleLastRunFailed || status == config.ReportScheduleLastRunRetrying || status == config.ReportScheduleLastRunUnconfirmed
}

// The caller owns scheduleRunMu across load, side effect and final persistence.
// Persisting running first makes a lost completion record visible after restart.
func (h *ReportingHandlers) executeReportSchedule(ctx context.Context, persistence *config.ConfigPersistence, store *config.ReportScheduleStore, index int, now time.Time, manual bool, key string, retry bool) (reportScheduleRunResult, config.ReportSchedule, error) {
	schedule := config.NormalizeReportSchedule(store.Schedules[index])
	if !retry {
		windowEnd := now.UTC().Truncate(time.Second)
		schedule.RunWindowEnd = &windowEnd
		schedule.RunManual = manual
		if manual && key == "" {
			key = currentReportScheduleOccurrenceKey(schedule, now)
		}
	}
	// An explicit operator retry starts a fresh bounded delivery budget.
	if !retry || manual {
		schedule.RunAttempts = 0
	}
	schedule.RunAttempts++
	schedule.LastRunStatus = config.ReportScheduleLastRunRunning
	schedule.LastDeliveryStatus = ""
	schedule.LastReportDataState = ""
	schedule.LastReportDataMessage = ""
	schedule.LastError = ""
	schedule.RetryAt = nil
	schedule.LastRunAt = &now
	schedule.UpdatedAt = now
	if key != "" {
		schedule.LastOccurrenceKey = key
	}
	if next, err := nextReportScheduleRunAt(schedule, now); err == nil {
		schedule.NextRunAt = &next
	}
	store.Schedules[index] = schedule
	if err := persistence.SaveReportScheduleStore(*store); err != nil {
		return reportScheduleRunResult{}, schedule, err
	}
	started := time.Now()
	result, updated := h.runReportSchedule(ctx, persistence, schedule, now, schedule.RunManual, "")
	completed := now.Add(time.Since(started)).UTC()
	updated.UpdatedAt = completed
	if updated.RetryAt != nil {
		retryAt := completed.Add(updated.RetryAt.Sub(now))
		updated.RetryAt = &retryAt
	}
	if updated.LastRunStatus == config.ReportScheduleLastRunOK {
		updated.LastSuccessAt = &completed
	}
	store.Schedules[index] = updated
	if err := persistence.SaveReportScheduleStore(*store); err != nil {
		return result, updated, err
	}
	h.recordReportingSuccessfulPeriod(ctx, updated, result.usagePeriod, completed)
	return result, updated, nil
}

func (h *ReportingHandlers) prepareReportSchedule(ctx context.Context, schedule config.ReportSchedule, now time.Time, statusOnly bool) (config.ReportSchedule, error) {
	schedule = config.NormalizeReportSchedule(schedule)
	if schedule.ID == "" {
		return schedule, reportScheduleValidationError{code: "invalid_schedule_id", message: "Schedule ID is required"}
	}
	if statusOnly {
		return schedule, nil
	}
	if schedule.Name == "" || len(schedule.Name) > 80 {
		return schedule, reportScheduleValidationError{code: "invalid_name", message: "Schedule name is required and must be 80 characters or fewer"}
	}
	switch schedule.Kind {
	case config.ReportScheduleKindResources:
	case config.ReportScheduleKindBackupHealth:
		for _, resource := range schedule.Scope.Resources {
			kind := reporting.CanonicalResourceType(resource.ResourceType)
			if kind != "vm" && kind != "system-container" {
				return schedule, reportScheduleValidationError{code: "invalid_scope", message: "Backup health reports support VMs and containers"}
			}
		}
	case config.ReportScheduleKindVMInventory:
		if schedule.Format == "" {
			schedule.Format = string(reporting.DescribeVMInventoryExport().Format)
		}
		if !reporting.DescribeVMInventoryExport().SupportsFormat(reporting.ReportFormat(schedule.Format)) {
			return schedule, reportScheduleValidationError{code: "invalid_format", message: "VM inventory schedules require CSV format"}
		}
		if len(schedule.Scope.Resources) > 0 || len(schedule.Scope.Tags) > 0 {
			return schedule, reportScheduleValidationError{code: "invalid_scope", message: "VM inventory includes all VMs in this organization. Resource and tag filters are not supported."}
		}
	case config.ReportScheduleKindPatrolDigest:
		prepared, err := preparePatrolDigestSchedule(schedule, now)
		if err != nil {
			return prepared, err
		}
		return prepared, h.validateEnabledReportDelivery(ctx, prepared)
	default:
		return schedule, reportScheduleValidationError{code: "invalid_kind", message: "Schedule kind must be resources, backup_health, vm_inventory or patrol_digest"}
	}
	if schedule.Format == "" {
		schedule.Format = config.ReportScheduleFormatPDF
	}
	if schedule.Format != config.ReportScheduleFormatPDF && schedule.Format != config.ReportScheduleFormatCSV {
		return schedule, reportScheduleValidationError{code: "invalid_format", message: "Schedule format must be pdf or csv"}
	}
	if schedule.Delivery.Method == "" {
		schedule.Delivery.Method = config.ReportScheduleDeliveryEmail
	}
	if schedule.Delivery.Method != config.ReportScheduleDeliveryEmail && schedule.Delivery.Method != config.ReportScheduleDeliveryDisk {
		return schedule, reportScheduleValidationError{code: "invalid_delivery", message: "Delivery method must be email or disk"}
	}
	if !schedule.Delivery.Attach && !schedule.Delivery.SaveToDisk {
		schedule.Delivery.Attach = true
		schedule.Delivery.SaveToDisk = true
	}
	// Resource reports have always been retained on disk, including email runs.
	schedule.Delivery.SaveToDisk = true
	if err := validateReportScheduleCadence(schedule.Cadence); err != nil {
		return schedule, err
	}
	if schedule.Kind == config.ReportScheduleKindResources {
		if err := h.validateReportScheduleScope(ctx, schedule.Scope); err != nil {
			return schedule, err
		}
	} else if schedule.Kind == config.ReportScheduleKindBackupHealth {
		if err := validateReportScheduleScopeFields(schedule.Scope); err != nil {
			return schedule, err
		}
	}
	next, err := nextReportScheduleRunAt(schedule, now)
	if err != nil {
		return schedule, err
	}
	schedule.NextRunAt = &next
	return schedule, h.validateEnabledReportDelivery(ctx, schedule)
}

func validateReportScheduleCadence(cadence config.ReportScheduleCadence) error {
	if cadence.Type != config.ReportScheduleCadenceMonthly && cadence.Type != config.ReportScheduleCadenceWeekly {
		return reportScheduleValidationError{code: "invalid_cadence", message: "Cadence must be monthly or weekly"}
	}
	if cadence.Type == config.ReportScheduleCadenceMonthly && (cadence.DayOfMonth < 1 || cadence.DayOfMonth > 28) {
		return reportScheduleValidationError{code: "invalid_day_of_month", message: "Monthly schedules must use a day from 1 to 28"}
	}
	if cadence.Type == config.ReportScheduleCadenceWeekly {
		if _, ok := parseReportScheduleWeekday(cadence.Weekday); !ok {
			return reportScheduleValidationError{code: "invalid_weekday", message: "Weekly schedules must include a weekday"}
		}
	}
	if _, err := time.Parse("15:04", cadence.Time); err != nil {
		return reportScheduleValidationError{code: "invalid_time", message: "Schedule time must use HH:MM"}
	}
	if _, err := reportScheduleLocation(cadence.Timezone); err != nil {
		return reportScheduleValidationError{code: "invalid_timezone", message: "Schedule timezone must be a valid IANA timezone"}
	}
	return nil
}

func (h *ReportingHandlers) validateReportScheduleScope(ctx context.Context, scope config.ReportScheduleScope) error {
	if len(scope.Resources) == 0 && len(scope.Tags) == 0 {
		return reportScheduleValidationError{code: "invalid_scope", message: "Select at least one resource or tag"}
	}
	definition := performanceReportDefinition()
	if len(scope.Resources) > definition.MultiResourceMax {
		return reportScheduleValidationError{code: "too_many_resources", message: fmt.Sprintf("Maximum %d resources allowed", definition.MultiResourceMax)}
	}
	if err := validateReportScheduleScopeFields(scope); err != nil {
		return err
	}
	if len(scope.Tags) > 0 {
		if _, err := h.resolveReportScheduleResources(ctx, GetOrgID(ctx), scope); err != nil {
			return err
		}
	}
	return nil
}

// Field validity is shared; scope cardinality and resolution belong to the
// report kind. Backup health intentionally supports empty and large estates.
func validateReportScheduleScopeFields(scope config.ReportScheduleScope) error {
	for _, res := range scope.Resources {
		if !validResourceID.MatchString(res.ResourceID) || len(res.ResourceID) > 128 {
			return reportScheduleValidationError{code: "invalid_resource_id", message: "Resource IDs must match [a-zA-Z0-9._:-]+ and be 128 characters or fewer"}
		}
		if _, err := normalizeReportResourceType(res.ResourceType); err != nil {
			return reportScheduleValidationError{code: "invalid_resource_type", message: err.Error()}
		}
	}
	for _, tag := range scope.Tags {
		if len(tag) > 64 || strings.ContainsAny(tag, "\r\n\t") {
			return reportScheduleValidationError{code: "invalid_tag", message: "Tags must be 64 characters or fewer and cannot contain control characters"}
		}
	}
	return nil
}

func (h *ReportingHandlers) resolveReportScheduleResources(ctx context.Context, orgID string, scope config.ReportScheduleScope) ([]multiReportResourceEntry, error) {
	definition := performanceReportDefinition()
	entries := make([]multiReportResourceEntry, 0, len(scope.Resources))
	seen := map[string]struct{}{}
	appendEntry := func(resourceType, resourceID string) error {
		resourceType, err := normalizeReportResourceType(resourceType)
		if err != nil {
			return err
		}
		key := resourceType + "\x00" + resourceID
		if _, exists := seen[key]; exists {
			return nil
		}
		seen[key] = struct{}{}
		entries = append(entries, multiReportResourceEntry{ResourceType: resourceType, ResourceID: resourceID})
		return nil
	}
	// A schedule reports only on resources its own organization holds, so
	// both explicit resources and tags resolve against that organization's
	// inventory.
	snapshot, ok := h.getReportingEnrichmentSnapshot(ctx, orgID)
	if !ok {
		return nil, reportScheduleValidationError{code: "scope_unavailable", message: "Schedule scope cannot be resolved until resource inventory is available"}
	}
	for _, res := range scope.Resources {
		if _, err := normalizeReportResourceType(res.ResourceType); err != nil {
			return nil, reportScheduleValidationError{code: "invalid_resource_type", message: err.Error()}
		}
		if !reportSubjectInInventory(res.ResourceID, snapshot) {
			// A resource removed since the schedule was saved, or never
			// part of this organization, is left out of the report.
			log.Warn().
				Str("org_id", orgID).
				Str("resource_id", res.ResourceID).
				Msg("report schedules: skipping a resource that is not in this organization's inventory")
			continue
		}
		if err := appendEntry(res.ResourceType, res.ResourceID); err != nil {
			return nil, reportScheduleValidationError{code: "invalid_resource_type", message: err.Error()}
		}
	}

	if len(scope.Tags) > 0 {
		tags := map[string]struct{}{}
		for _, tag := range scope.Tags {
			tags[strings.ToLower(strings.TrimSpace(tag))] = struct{}{}
		}
		for _, resource := range snapshot.Resources {
			if !resourceHasAnyReportScheduleTag(resource, tags) {
				continue
			}
			resourceType := reporting.CanonicalResourceType(string(resource.Type))
			if resourceType == "" || strings.TrimSpace(resource.ID) == "" {
				continue
			}
			if err := appendEntry(resourceType, resource.ID); err != nil {
				continue
			}
		}
	}
	if len(entries) == 0 {
		return nil, reportScheduleValidationError{code: "empty_scope", message: "Schedule scope did not match any reportable resources"}
	}
	if len(entries) > definition.MultiResourceMax {
		return nil, reportScheduleValidationError{code: "too_many_resources", message: fmt.Sprintf("Maximum %d resources allowed", definition.MultiResourceMax)}
	}
	return entries, nil
}

func resourceHasAnyReportScheduleTag(resource unifiedresources.Resource, tags map[string]struct{}) bool {
	for _, tag := range resource.Tags {
		if _, ok := tags[strings.ToLower(strings.TrimSpace(tag))]; ok {
			return true
		}
	}
	return false
}

type reportScheduleRunResult struct {
	path        string
	email       string
	usagePeriod string
}

func (h *ReportingHandlers) runReportSchedule(ctx context.Context, persistence *config.ConfigPersistence, schedule config.ReportSchedule, now time.Time, manual bool, occurrenceKey string) (reportScheduleRunResult, config.ReportSchedule) {
	schedule = config.NormalizeReportSchedule(schedule)
	result := reportScheduleRunResult{}
	if schedule.RunAttempts < 1 {
		schedule.RunAttempts = 1
	}
	schedule.LastRunAt = &now
	schedule.UpdatedAt = now
	if occurrenceKey != "" {
		schedule.LastOccurrenceKey = occurrenceKey
	}
	if next, err := nextReportScheduleRunAt(schedule, now); err == nil {
		schedule.NextRunAt = &next
	}
	if err := h.requireCommercialFeature(ctx, featureAdvancedReportingValue); err != nil {
		return result, markReportScheduleFailed(schedule, now, fmt.Errorf("advanced reporting entitlement required: %w", err))
	}
	if schedule.Kind == config.ReportScheduleKindPatrolDigest {
		return h.runPatrolDigestSchedule(ctx, persistence, schedule, now)
	}
	if schedule.Delivery.Method == config.ReportScheduleDeliveryEmail {
		if _, err := scheduledReportEmailManager(persistence, schedule); err != nil {
			failed := markReportScheduleDeliveryFailed(schedule, now, err)
			h.recordReportingDeliveryFailure(schedule.Kind, failed.LastDeliveryStatus, now)
			return result, failed
		}
	}

	report, err := h.generateScheduledFileReport(ctx, schedule, now, manual)
	if err != nil {
		return result, markReportScheduleFailed(schedule, now, err)
	}
	h.recordReportingActivity(config.ReportingActivityEvent{At: now, Kind: schedule.Kind, Event: "generated", DataState: report.DataState})
	result.usagePeriod = reportingActivityPeriod(schedule, report.Start, report.End, now)

	path, err := saveGeneratedReport(persistence, schedule, report)
	if err != nil {
		return result, markReportScheduleFailed(schedule, now, err)
	}
	result.path = path
	h.recordReportingActivity(config.ReportingActivityEvent{At: now, Kind: schedule.Kind, Event: "saved"})
	schedule.LastReportDataState = report.DataState
	schedule.LastReportDataMessage = report.DataMessage
	schedule.LastReportPath = path
	schedule.LastDeliveryStatus = "saved"

	if schedule.Delivery.Method == config.ReportScheduleDeliveryEmail {
		emailStatus, err := sendScheduledReportEmail(persistence, schedule, report, path)
		if err != nil {
			failed := markReportScheduleDeliveryFailed(schedule, now, err)
			h.recordReportingDeliveryFailure(schedule.Kind, failed.LastDeliveryStatus, now)
			return result, failed
		}
		result.email = emailStatus
		schedule.LastDeliveryStatus = emailStatus
		h.recordReportingActivity(config.ReportingActivityEvent{At: now, Kind: schedule.Kind, Event: "email_accepted"})
	}

	schedule.LastRunStatus = config.ReportScheduleLastRunOK
	schedule.LastError = ""
	schedule.RetryAt = nil
	schedule.LastSuccessAt = &now
	return result, schedule
}

func (h *ReportingHandlers) generateScheduledFileReport(ctx context.Context, schedule config.ReportSchedule, now time.Time, manual bool) (generatedMultiReport, error) {
	if schedule.Kind == config.ReportScheduleKindBackupHealth {
		return h.generateBackupHealthReport(ctx, schedule, now, manual)
	}
	orgID := GetOrgID(ctx)
	if schedule.Kind == config.ReportScheduleKindVMInventory {
		snapshot, ok := h.getReportingEnrichmentSnapshot(ctx, orgID)
		if !ok {
			return generatedMultiReport{}, errors.New(vmInventoryUnavailableMessage)
		}
		return generateVMInventoryReport(snapshot, now)
	}
	resources, err := h.resolveReportScheduleResources(ctx, orgID, schedule.Scope)
	if err != nil {
		return generatedMultiReport{}, err
	}
	windowEnd := now
	if schedule.RunWindowEnd != nil {
		windowEnd = *schedule.RunWindowEnd
	}
	start, end, err := reportScheduleWindow(schedule, windowEnd, manual)
	if err != nil {
		return generatedMultiReport{}, err
	}
	return h.generateMultiReportFromBody(ctx, multiReportRequestBody{
		Resources: resources, Format: schedule.Format,
		Start: start.UTC().Format(time.RFC3339), End: end.UTC().Format(time.RFC3339), Title: schedule.Name,
	}, now)
}

func markReportScheduleFailed(schedule config.ReportSchedule, now time.Time, err error) config.ReportSchedule {
	schedule.LastRunStatus = config.ReportScheduleLastRunFailed
	schedule.LastError = strings.TrimSpace(err.Error())
	schedule.LastRunAt = &now
	schedule.UpdatedAt = now
	schedule.RetryAt = nil
	return schedule
}

func markReportScheduleDeliveryFailed(schedule config.ReportSchedule, now time.Time, err error) config.ReportSchedule {
	schedule = markReportScheduleFailed(schedule, now, err)
	schedule.LastDeliveryStatus = "failed"
	var unconfirmed *notifications.EmailAcceptanceUnconfirmedError
	if errors.As(err, &unconfirmed) {
		schedule.LastRunStatus = config.ReportScheduleLastRunUnconfirmed
		schedule.LastDeliveryStatus = "unconfirmed"
		schedule.LastError = "The mail server may have accepted the report. Check the recipient before retrying to avoid a duplicate."
		return schedule
	}
	class := notifications.ClassifyNotificationFailureError(err)
	switch class {
	case notifications.NotificationFailureAuthentication:
		schedule.LastError = "Email authentication failed. Check the credentials in Alerts > Notifications."
	case notifications.NotificationFailureConfiguration:
		var setup reportScheduleValidationError
		if errors.As(err, &setup) {
			schedule.LastError = setup.message
		} else {
			schedule.LastError = "Email configuration was rejected. Check Alerts > Notifications."
		}
	case notifications.NotificationFailureRejected:
		schedule.LastError = "The mail server rejected this report. Check the recipient and notification settings."
	case notifications.NotificationFailureTLS:
		schedule.LastError = "The secure mail connection failed. Check the server certificate and TLS settings in Alerts > Notifications."
	default:
		schedule.LastError = "The mail server is temporarily unavailable. Check the connection and notification settings."
	}
	if class.Retryable() && schedule.Enabled && schedule.RunAttempts < reportScheduleMaxAttempts {
		delay := time.Minute
		if schedule.RunAttempts >= 2 {
			delay = 5 * time.Minute
		}
		retryAt := now.Add(delay)
		schedule.RetryAt = &retryAt
		schedule.LastRunStatus = config.ReportScheduleLastRunRetrying
	} else if class.Retryable() && schedule.RunAttempts >= reportScheduleMaxAttempts {
		schedule.LastError = "Delivery failed after 3 attempts. " + schedule.LastError
	}
	return schedule
}

func reportScheduleWindow(schedule config.ReportSchedule, now time.Time, manual bool) (time.Time, time.Time, error) {
	location, err := reportScheduleLocation(schedule.Cadence.Timezone)
	if err != nil {
		return time.Time{}, time.Time{}, err
	}
	end := now.In(location)
	if !manual {
		occurrence, _, err := lastReportScheduleOccurrenceAt(schedule, now)
		if err == nil {
			end = occurrence.In(location)
		}
	}
	switch schedule.Cadence.Type {
	case config.ReportScheduleCadenceMonthly:
		firstThisMonth := time.Date(end.Year(), end.Month(), 1, 0, 0, 0, 0, location)
		firstPreviousMonth := firstThisMonth.AddDate(0, -1, 0)
		return firstPreviousMonth.UTC(), firstThisMonth.UTC(), nil
	case config.ReportScheduleCadenceWeekly:
		return end.AddDate(0, 0, -7).UTC(), end.UTC(), nil
	default:
		return time.Time{}, time.Time{}, fmt.Errorf("unsupported cadence %q", schedule.Cadence.Type)
	}
}

func saveGeneratedReport(persistence *config.ConfigPersistence, schedule config.ReportSchedule, report generatedMultiReport) (string, error) {
	baseDir, err := securityutil.JoinStorageLeaf(persistence.DataDir(), "reports")
	if err != nil {
		return "", err
	}
	generatedDir, err := securityutil.JoinStorageLeaf(baseDir, "generated")
	if err != nil {
		return "", err
	}
	scheduleDir, err := securityutil.JoinStorageLeaf(generatedDir, securityutil.HashedStorageName(schedule.ID))
	if err != nil {
		return "", err
	}
	if err := os.MkdirAll(scheduleDir, 0o700); err != nil {
		return "", fmt.Errorf("create report output directory: %w", err)
	}
	filename := sanitizeFilename(report.Filename)
	if filename == "" {
		filename = "report." + string(report.Format)
	}
	path, err := securityutil.JoinStorageLeaf(scheduleDir, filename)
	if err != nil {
		return "", err
	}
	if err := os.WriteFile(path, report.Data, 0o600); err != nil {
		return "", fmt.Errorf("write generated report: %w", err)
	}
	pruneGeneratedReports(scheduleDir, schedule.RetentionCount)
	return path, nil
}

func pruneGeneratedReports(dir string, retentionCount int) {
	if retentionCount <= 0 {
		retentionCount = config.DefaultReportScheduleRetentionCount
	}
	entries, err := os.ReadDir(dir)
	if err != nil {
		return
	}
	type generatedFile struct {
		path    string
		modTime time.Time
	}
	files := make([]generatedFile, 0, len(entries))
	for _, entry := range entries {
		if entry.IsDir() {
			continue
		}
		info, err := entry.Info()
		if err != nil {
			continue
		}
		path, err := securityutil.JoinStorageLeaf(dir, entry.Name())
		if err != nil {
			continue
		}
		files = append(files, generatedFile{path: path, modTime: info.ModTime()})
	}
	sort.Slice(files, func(i, j int) bool {
		return files[i].modTime.After(files[j].modTime)
	})
	for i := retentionCount; i < len(files); i++ {
		_ = os.Remove(files[i].path)
	}
}

func reportEmailConfigurationFailure(message string) error {
	return notifications.FailWithClass(notifications.NotificationFailureConfiguration,
		reportScheduleValidationError{code: "email_not_ready", message: message})
}

// Configuration validation and SMTP semantics remain owned by notifications.
// Reporting persists the retry budget, so one attempt must be one SMTP send.
func scheduledReportEmailManager(persistence *config.ConfigPersistence, schedule config.ReportSchedule) (*notifications.EnhancedEmailManager, error) {
	if persistence == nil {
		return nil, reportEmailConfigurationFailure("Email configuration is unavailable. Check Alerts > Notifications.")
	}
	emailCfg, err := persistence.LoadEmailConfig()
	if err != nil {
		return nil, reportEmailConfigurationFailure("Email configuration could not be loaded. Check notification settings and server storage.")
	}
	if emailCfg == nil || !emailCfg.Enabled {
		return nil, reportEmailConfigurationFailure("Enable email notifications in Alerts > Notifications before running this report.")
	}
	recipients := schedule.Delivery.To
	if len(recipients) == 0 {
		recipients = emailCfg.To
	}
	if len(recipients) == 0 && strings.TrimSpace(emailCfg.From) != "" {
		recipients = []string{emailCfg.From}
	}
	if len(recipients) == 0 {
		return nil, reportEmailConfigurationFailure("Add report recipients or email notification recipients in Alerts > Notifications.")
	}
	providerConfig := notifications.EmailProviderConfig{
		EmailConfig: notifications.EmailConfig{
			Provider:  emailCfg.Provider,
			From:      emailCfg.From,
			To:        recipients,
			SMTPHost:  emailCfg.SMTPHost,
			SMTPPort:  emailCfg.SMTPPort,
			Username:  emailCfg.Username,
			Password:  emailCfg.Password,
			TLS:       emailCfg.TLS,
			StartTLS:  emailCfg.StartTLS,
			RateLimit: emailCfg.RateLimit,
		},
		Provider:     emailCfg.Provider,
		MaxRetries:   0,
		RateLimit:    emailCfg.RateLimit,
		StartTLS:     emailCfg.StartTLS,
		AuthRequired: emailCfg.Username != "" && emailCfg.Password != "",
	}
	manager := notifications.NewEnhancedEmailManager(providerConfig)
	if err := manager.ValidateConfiguration(); err != nil {
		return nil, reportEmailConfigurationFailure(err.Error())
	}
	return manager, nil
}

func sendScheduledReportEmail(persistence *config.ConfigPersistence, schedule config.ReportSchedule, report generatedMultiReport, path string) (string, error) {
	manager, err := scheduledReportEmailManager(persistence, schedule)
	if err != nil {
		return "", err
	}
	subject := "Pulse report: " + schedule.Name
	htmlBody := "<p>Your scheduled Pulse report is ready.</p>"
	textBody := "Your scheduled Pulse report is ready."
	attachments := []notifications.EmailAttachment{}
	if schedule.Delivery.Attach && len(report.Data) <= reportScheduleAttachmentLimitBytes {
		attachments = append(attachments, notifications.EmailAttachment{
			Filename:    report.Filename,
			ContentType: report.ContentType,
			Data:        report.Data,
		})
	} else if schedule.Delivery.Attach && len(report.Data) > reportScheduleAttachmentLimitBytes {
		return "", reportEmailConfigurationFailure("The report exceeds the 15 MiB attachment limit. Choose CSV or turn off Attach report to email a saved-path notice.")
	} else {
		htmlBody = "<p>Your scheduled Pulse report was generated and saved to disk.</p><p>Saved path: " + htmlEscape(path) + "</p>"
		textBody = "Your scheduled Pulse report was generated and saved to disk.\nSaved path: " + path
	}
	if err := manager.SendEmailWithAttachments(subject, htmlBody, textBody, attachments); err != nil {
		return "", err
	}
	if len(attachments) > 0 {
		return "sent_with_attachment", nil
	}
	return "sent_saved_path", nil
}

func htmlEscape(value string) string {
	replacer := strings.NewReplacer("&", "&amp;", "<", "&lt;", ">", "&gt;", "\"", "&quot;", "'", "&#39;")
	return replacer.Replace(value)
}

func reportScheduleLocation(name string) (*time.Location, error) {
	name = strings.TrimSpace(name)
	if name == "" || name == "Local" {
		name = "UTC"
	}
	return time.LoadLocation(name)
}

func parseReportScheduleClock(clock string) (hour, minute int, err error) {
	parsed, err := time.Parse("15:04", strings.TrimSpace(clock))
	if err != nil {
		return 0, 0, err
	}
	return parsed.Hour(), parsed.Minute(), nil
}

func parseReportScheduleWeekday(value string) (time.Weekday, bool) {
	switch strings.ToLower(strings.TrimSpace(value)) {
	case "sunday", "sun":
		return time.Sunday, true
	case "monday", "mon":
		return time.Monday, true
	case "tuesday", "tue":
		return time.Tuesday, true
	case "wednesday", "wed":
		return time.Wednesday, true
	case "thursday", "thu":
		return time.Thursday, true
	case "friday", "fri":
		return time.Friday, true
	case "saturday", "sat":
		return time.Saturday, true
	default:
		return time.Sunday, false
	}
}

func nextReportScheduleRunAt(schedule config.ReportSchedule, after time.Time) (time.Time, error) {
	location, err := reportScheduleLocation(schedule.Cadence.Timezone)
	if err != nil {
		return time.Time{}, err
	}
	hour, minute, err := parseReportScheduleClock(schedule.Cadence.Time)
	if err != nil {
		return time.Time{}, err
	}
	localAfter := after.In(location)
	switch schedule.Cadence.Type {
	case config.ReportScheduleCadenceMonthly:
		candidate := time.Date(localAfter.Year(), localAfter.Month(), schedule.Cadence.DayOfMonth, hour, minute, 0, 0, location)
		if !candidate.After(localAfter) {
			candidate = candidate.AddDate(0, 1, 0)
		}
		return candidate.UTC(), nil
	case config.ReportScheduleCadenceWeekly:
		weekday, ok := parseReportScheduleWeekday(schedule.Cadence.Weekday)
		if !ok {
			return time.Time{}, fmt.Errorf("invalid weekday")
		}
		dayDelta := (int(weekday) - int(localAfter.Weekday()) + 7) % 7
		candidateDate := localAfter.AddDate(0, 0, dayDelta)
		candidate := time.Date(candidateDate.Year(), candidateDate.Month(), candidateDate.Day(), hour, minute, 0, 0, location)
		if !candidate.After(localAfter) {
			candidate = candidate.AddDate(0, 0, 7)
		}
		return candidate.UTC(), nil
	default:
		return time.Time{}, fmt.Errorf("unsupported cadence %q", schedule.Cadence.Type)
	}
}

func lastReportScheduleOccurrenceAt(schedule config.ReportSchedule, now time.Time) (time.Time, string, error) {
	location, err := reportScheduleLocation(schedule.Cadence.Timezone)
	if err != nil {
		return time.Time{}, "", err
	}
	hour, minute, err := parseReportScheduleClock(schedule.Cadence.Time)
	if err != nil {
		return time.Time{}, "", err
	}
	localNow := now.In(location)
	var occurrence time.Time
	switch schedule.Cadence.Type {
	case config.ReportScheduleCadenceMonthly:
		occurrence = time.Date(localNow.Year(), localNow.Month(), schedule.Cadence.DayOfMonth, hour, minute, 0, 0, location)
		if occurrence.After(localNow) {
			occurrence = occurrence.AddDate(0, -1, 0)
		}
	case config.ReportScheduleCadenceWeekly:
		weekday, ok := parseReportScheduleWeekday(schedule.Cadence.Weekday)
		if !ok {
			return time.Time{}, "", fmt.Errorf("invalid weekday")
		}
		dayDelta := (int(localNow.Weekday()) - int(weekday) + 7) % 7
		occurrenceDate := localNow.AddDate(0, 0, -dayDelta)
		occurrence = time.Date(occurrenceDate.Year(), occurrenceDate.Month(), occurrenceDate.Day(), hour, minute, 0, 0, location)
		if occurrence.After(localNow) {
			occurrence = occurrence.AddDate(0, 0, -7)
		}
	default:
		return time.Time{}, "", fmt.Errorf("unsupported cadence %q", schedule.Cadence.Type)
	}
	return occurrence.UTC(), occurrenceKey(schedule, occurrence), nil
}

func occurrenceKey(schedule config.ReportSchedule, occurrence time.Time) string {
	timezone := strings.TrimSpace(schedule.Cadence.Timezone)
	if timezone == "" {
		timezone = "UTC"
	}
	return schedule.Cadence.Type + ":" + occurrence.UTC().Format("2006-01-02T15:04:05Z07:00") + ":" + timezone
}

func (h *ReportingHandlers) RunReportScheduleScheduler(ctx context.Context) {
	if h == nil {
		return
	}
	h.runDueReportSchedules(ctx, time.Now().UTC())
	ticker := time.NewTicker(reportScheduleTickInterval)
	defer ticker.Stop()
	for {
		select {
		case <-ctx.Done():
			return
		case now := <-ticker.C:
			h.runDueReportSchedules(ctx, now.UTC())
		}
	}
}

func (h *ReportingHandlers) runDueReportSchedules(ctx context.Context, now time.Time) {
	manager := h.reportMonitorManager()
	if manager == nil {
		return
	}
	orgIDs, err := manager.ListOrganizationIDs()
	if err != nil {
		log.Warn().Err(err).Msg("report schedules: list organizations")
		return
	}
	for _, orgID := range orgIDs {
		orgCtx := context.WithValue(ctx, OrgIDContextKey, orgID)
		h.syncCommercialHistoryRetention(orgCtx, orgID, now)
		h.runDueReportSchedulesForOrg(orgCtx, orgID, now)
	}
}

func (h *ReportingHandlers) syncCommercialHistoryRetention(ctx context.Context, orgID string, now time.Time) {
	manager := h.reportMonitorManager()
	if manager == nil || h.commercialLicenseResolver == nil {
		return
	}
	monitor, err := manager.GetMonitor(orgID)
	if err != nil || monitor == nil || monitor.GetMetricsStore() == nil {
		return
	}
	service := h.commercialLicenseResolver(ctx)
	if service == nil {
		monitor.GetMetricsStore().SetCommercialHistoryRetention(0, time.Time{})
		return
	}
	activation := service.GetActivationState()
	if activation == nil || activation.Continuity.DowngradeRetention == nil {
		monitor.GetMetricsStore().SetCommercialHistoryRetention(0, time.Time{})
		return
	}
	downgrade := normalizeDowngradeRetentionStateFromLicensing(*activation.Continuity.DowngradeRetention)
	monitor.GetMetricsStore().SetCommercialHistoryRetention(
		downgrade.CurrentHistoryDays,
		time.Unix(downgrade.PurgeEligibleAt, 0).UTC(),
	)
	if now.UTC().Unix() < downgrade.PurgeEligibleAt || service.RequireFeature(featureAdvancedReportingValue) == nil {
		return
	}
	persistence, err := h.reportSchedulePersistence(ctx)
	if err != nil {
		return
	}
	if err := purgeGeneratedReportsAfterDowngrade(persistence); err != nil {
		log.Warn().Err(err).Str("org_id", orgID).Msg("report schedules: purge downgrade artifacts")
	}
}

func purgeGeneratedReportsAfterDowngrade(persistence *config.ConfigPersistence) error {
	if persistence == nil {
		return nil
	}
	baseDir, err := securityutil.JoinStorageLeaf(persistence.DataDir(), "reports")
	if err != nil {
		return err
	}
	generatedDir, err := securityutil.JoinStorageLeaf(baseDir, "generated")
	if err != nil {
		return err
	}
	scheduleDirs, err := os.ReadDir(generatedDir)
	if errors.Is(err, os.ErrNotExist) {
		return nil
	}
	if err != nil {
		return err
	}
	for _, scheduleDir := range scheduleDirs {
		if !scheduleDir.IsDir() || scheduleDir.Type()&os.ModeSymlink != 0 {
			continue
		}
		dir, err := securityutil.JoinStorageLeaf(generatedDir, scheduleDir.Name())
		if err != nil {
			continue
		}
		files, err := os.ReadDir(dir)
		if err != nil {
			continue
		}
		for _, file := range files {
			if file.IsDir() || file.Type()&os.ModeSymlink != 0 {
				continue
			}
			path, err := securityutil.JoinStorageLeaf(dir, file.Name())
			if err == nil {
				_ = os.Remove(path)
			}
		}
	}
	return nil
}

func (h *ReportingHandlers) runDueReportSchedulesForOrg(ctx context.Context, orgID string, now time.Time) {
	h.scheduleRunMu.Lock()
	defer h.scheduleRunMu.Unlock()
	persistence, store, err := h.reportSchedulePersistenceAndStore(ctx)
	if err != nil {
		log.Warn().Err(err).Str("org_id", orgID).Msg("report schedules: load store")
		return
	}
	save := func(index int, schedule config.ReportSchedule) bool {
		store.Schedules[index] = schedule
		if err := persistence.SaveReportScheduleStore(*store); err != nil {
			log.Warn().Err(err).Str("org_id", orgID).Msg("report schedules: save status")
			return false
		}
		return true
	}
	for i := range store.Schedules {
		if ctx.Err() != nil {
			return
		}
		schedule := config.NormalizeReportSchedule(store.Schedules[i])
		if schedule.LastRunStatus == config.ReportScheduleLastRunRunning {
			schedule.UpdatedAt = now
			schedule.RetryAt = nil
			schedule.LastRunStatus = config.ReportScheduleLastRunFailed
			schedule.LastError = "The previous report run ended before completion was recorded. Run it again."
			if schedule.Delivery.Method == config.ReportScheduleDeliveryEmail {
				schedule.LastRunStatus = config.ReportScheduleLastRunUnconfirmed
				schedule.LastDeliveryStatus = "unconfirmed"
				schedule.LastError = "The previous email run ended before delivery was recorded. Check the recipient before retrying to avoid a duplicate."
			} else if schedule.Enabled && schedule.RunWindowEnd != nil && schedule.RunAttempts < reportScheduleMaxAttempts {
				schedule.LastRunStatus = config.ReportScheduleLastRunRetrying
				schedule.LastError = "Report generation was interrupted. Retrying the saved report window."
				schedule.RetryAt = &now
			}
			if !save(i, schedule) {
				return
			}
			if schedule.LastRunStatus == config.ReportScheduleLastRunUnconfirmed {
				h.recordReportingDeliveryFailure(schedule.Kind, schedule.LastDeliveryStatus, now)
			}
		}
		if !schedule.Enabled {
			if schedule.RetryAt != nil {
				schedule.RetryAt = nil
				schedule.RunWindowEnd = nil
				schedule.LastRunStatus = config.ReportScheduleLastRunFailed
				schedule.LastError = "Automatic retry cancelled because this schedule is disabled."
				schedule.UpdatedAt = now
				if !save(i, schedule) {
					return
				}
			}
			continue
		}
		if schedule.RetryAt != nil {
			if schedule.LastRunStatus != config.ReportScheduleLastRunRetrying || schedule.RunWindowEnd == nil || schedule.RunAttempts >= reportScheduleMaxAttempts {
				schedule.RetryAt = nil
				schedule.LastRunStatus = config.ReportScheduleLastRunFailed
				schedule.LastError = "The saved retry is incomplete or exhausted. Run a new report."
				if !save(i, schedule) {
					return
				}
				continue
			}
			if now.Before(*schedule.RetryAt) {
				continue
			}
			if _, _, err := h.executeReportSchedule(ctx, persistence, store, i, now, false, "", true); err != nil {
				log.Warn().Err(err).Str("org_id", orgID).Msg("report schedules: persist retry outcome")
				return
			}
			continue
		}
		occurrence, key, err := lastReportScheduleOccurrenceAt(schedule, now)
		if err != nil {
			if !save(i, markReportScheduleFailed(schedule, now, err)) {
				return
			}
			continue
		}
		if key == "" || key == schedule.LastOccurrenceKey || occurrence.After(now) {
			continue
		}
		if now.Sub(occurrence) > reportScheduleMissedRunGrace {
			schedule.LastOccurrenceKey = key
			if next, err := nextReportScheduleRunAt(schedule, now); err == nil {
				schedule.NextRunAt = &next
			}
			schedule.UpdatedAt = now
			if !save(i, schedule) {
				return
			}
			continue
		}
		if _, _, err := h.executeReportSchedule(ctx, persistence, store, i, now, false, key, false); err != nil {
			log.Warn().Err(err).Str("org_id", orgID).Msg("report schedules: persist outcome")
			return
		}
	}
}

// preparePatrolDigestSchedule validates the Patrol weekly summary kind. The
// summary is fleet-wide and only makes sense as a weekly email, so scope,
// format, attachments, and disk delivery are fixed rather than user choices.
func preparePatrolDigestSchedule(schedule config.ReportSchedule, now time.Time) (config.ReportSchedule, error) {
	schedule.Format = config.ReportScheduleFormatEmail
	schedule.Scope = config.ReportScheduleScope{Resources: []config.ReportScheduleResource{}, Tags: []string{}}
	if schedule.Delivery.Method == "" {
		schedule.Delivery.Method = config.ReportScheduleDeliveryEmail
	}
	if schedule.Delivery.Method != config.ReportScheduleDeliveryEmail {
		return schedule, reportScheduleValidationError{code: "invalid_delivery", message: "Patrol weekly summaries are delivered by email"}
	}
	schedule.Delivery.Attach = false
	schedule.Delivery.SaveToDisk = false
	if schedule.Cadence.Type != config.ReportScheduleCadenceWeekly {
		return schedule, reportScheduleValidationError{code: "invalid_cadence", message: "Patrol weekly summaries use a weekly cadence"}
	}
	if err := validateReportScheduleCadence(schedule.Cadence); err != nil {
		return schedule, err
	}
	next, err := nextReportScheduleRunAt(schedule, now)
	if err != nil {
		return schedule, err
	}
	schedule.NextRunAt = &next
	return schedule, nil
}

// runPatrolDigestSchedule emails the last seven days of Patrol work. Nothing is
// written to disk: the digest is a rollup of records Pulse already keeps, so
// the email is the only artifact.
func (h *ReportingHandlers) runPatrolDigestSchedule(ctx context.Context, persistence *config.ConfigPersistence, schedule config.ReportSchedule, now time.Time) (reportScheduleRunResult, config.ReportSchedule) {
	result := reportScheduleRunResult{}
	if h == nil || h.patrolDigestResolver == nil {
		return result, markReportScheduleFailed(schedule, now, errors.New("Patrol weekly summary is unavailable on this server"))
	}
	digest, ok := h.patrolDigestResolver(ctx, ai.PatrolDigestDefaultDays)
	if !ok {
		return result, markReportScheduleFailed(schedule, now, errors.New("Patrol is not available for this workspace"))
	}
	manager, err := scheduledReportEmailManager(persistence, schedule)
	if err != nil {
		failed := markReportScheduleDeliveryFailed(schedule, now, err)
		h.recordReportingDeliveryFailure(schedule.Kind, failed.LastDeliveryStatus, now)
		return result, failed
	}
	subject, htmlBody, textBody := renderPatrolDigestEmail(digest, h.patrolDigestPublicURL())
	h.recordReportingActivity(config.ReportingActivityEvent{At: now, Kind: schedule.Kind, Event: "generated"})
	result.usagePeriod = reportingActivityPeriod(schedule, time.Time{}, time.Time{}, now)
	if err := manager.SendEmailWithRetry(subject, htmlBody, textBody); err != nil {
		failed := markReportScheduleDeliveryFailed(schedule, now, err)
		h.recordReportingDeliveryFailure(schedule.Kind, failed.LastDeliveryStatus, now)
		return result, failed
	}
	h.recordReportingActivity(config.ReportingActivityEvent{At: now, Kind: schedule.Kind, Event: "email_accepted"})
	result.email = "sent"
	schedule.LastRunStatus = config.ReportScheduleLastRunOK
	schedule.LastError = ""
	schedule.LastDeliveryStatus = result.email
	schedule.RetryAt = nil
	schedule.LastSuccessAt = &now
	return result, schedule
}

func (h *ReportingHandlers) patrolDigestPublicURL() string {
	if h == nil || h.settingsStore == nil {
		return ""
	}
	settings, err := h.settingsStore.LoadSystemSettings()
	if err != nil || settings == nil {
		return ""
	}
	return strings.TrimRight(strings.TrimSpace(settings.PublicURL), "/")
}