Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

10.3 KB
Raw Blame
package api

import (
	"net/http"
	"strings"
	"time"

	"github.com/rs/zerolog/log"

	"github.com/rcourtman/pulse-go-rewrite/internal/config"
	"github.com/rcourtman/pulse-go-rewrite/internal/mock"
	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/internal/recovery"
	proxmoxrecoverymapper "github.com/rcourtman/pulse-go-rewrite/internal/recovery/mapper/proxmox"
	"github.com/rcourtman/pulse-go-rewrite/internal/recovery/model"
	"github.com/rcourtman/pulse-go-rewrite/internal/utils"
)

// BackupJobsResponse is the payload for `GET /api/backups/jobs`: PVE backup
// job outcomes for the last 24 hours and for each of the last 14 local days,
// summarized by recovery.BuildBackupHealth from retained task history (the
// same authority as the scheduled backup health report). The live
// /api/backups/pve task list only spans the latest collection window, so it
// cannot answer "is this normal compared with the last two weeks".
type BackupJobsResponse struct {
	Data BackupJobsPayload `json:"data"`
}

type BackupJobsPayload struct {
	// complete, partial, not_observed or unavailable, over the 14 days.
	HistoryState string           `json:"historyState"`
	Timezone     string           `json:"timezone"`
	Last24h      BackupJobsWindow `json:"last24h"`
	Days         []BackupJobsDay  `json:"days"`
}

type BackupJobsCounts struct {
	Success int `json:"success"`
	Failed  int `json:"failed"`
	Warning int `json:"warning"`
	Running int `json:"running"`
	Unknown int `json:"unknown"`
}

type BackupJobsWindow struct {
	Start        time.Time           `json:"start"`
	End          time.Time           `json:"end"`
	HistoryState string              `json:"historyState"`
	Counts       BackupJobsCounts    `json:"counts"`
	Failures     []BackupJobsFailure `json:"failures"`
}

// BackupJobsFailure names the workload only. Provider error text stays in
// the task drill-down, outside this summary (BACKUP_TASK_TIMELINE_CONTRACT.md).
type BackupJobsFailure struct {
	WorkloadID   string    `json:"workloadId"`
	WorkloadName string    `json:"workloadName"`
	Instance     string    `json:"instance"`
	At           time.Time `json:"at"`
	Outcome      string    `json:"outcome"`
}

type BackupJobsDay struct {
	Date         string           `json:"date"`
	HistoryState string           `json:"historyState"`
	Counts       BackupJobsCounts `json:"counts"`
}

const backupJobsHistoryDays = 14

func (r *Router) handleBackupJobs(w http.ResponseWriter, req *http.Request) {
	if req.Method != http.MethodGet {
		writeErrorResponse(w, http.StatusMethodNotAllowed, "method_not_allowed",
			"Only GET method is allowed", nil)
		return
	}
	monitor := r.getTenantMonitor(req.Context())
	if monitor == nil {
		writeErrorResponse(w, http.StatusInternalServerError, "no_monitor",
			"Monitor not available", nil)
		return
	}

	// Day boundaries follow the viewer's IANA zone; anything unreadable is UTC.
	location := time.UTC
	if name := strings.TrimSpace(req.URL.Query().Get("timezone")); name != "" {
		if loaded, err := time.LoadLocation(name); err == nil {
			location = loaded
		}
	}
	now := time.Now().UTC()
	from := backupJobsHistoryStart(now, location)

	resources, _ := monitor.UnifiedResourceSnapshot()
	all, err := backupHealthWorkloads(reportingEnrichmentSnapshot{Resources: resources}, config.ReportScheduleScope{})
	if err != nil {
		writeErrorResponse(w, http.StatusInternalServerError, "workloads_unavailable",
			"Backup job workloads are unavailable", nil)
		return
	}
	workloads := pveBackupJobWorkloads(all)

	var points []recovery.RecoveryPoint
	var coverage model.BackupHistoryCoverage
	available := true
	if mock.IsMockEnabled() {
		points, coverage = mockBackupJobHistory(monitor.PVEBackupsSnapshot().BackupTasks, workloads, from, now)
	} else if r.recoveryHandlers == nil {
		available = false
	} else if store, storeErr := r.recoveryHandlers.storeForOrg(GetOrgID(req.Context())); storeErr != nil {
		available = false
		log.Warn().Err(storeErr).Msg("Backup job history store unavailable")
	} else if points, coverage, err = store.BackupHealthHistory(req.Context(), from, now); err != nil {
		available = false
		log.Warn().Err(err).Msg("Backup job history unavailable")
	}

	payload := buildBackupJobsPayload(workloads, points, coverage, available, from, now, location)

	if err := utils.WriteJSONResponse(w, BackupJobsResponse{Data: payload}); err != nil {
		log.Error().Err(err).Msg("Failed to encode backup jobs response")
	}
}

// backupJobsHistoryStart is local midnight 13 days before today, so the
// window holds 14 whole local days ending now.
func backupJobsHistoryStart(now time.Time, location *time.Location) time.Time {
	local := now.In(location)
	return time.Date(local.Year(), local.Month(), local.Day(), 0, 0, 0, 0, location).
		AddDate(0, 0, -(backupJobsHistoryDays - 1)).UTC()
}

func buildBackupJobsPayload(
	workloads []model.BackupHealthWorkload,
	points []recovery.RecoveryPoint,
	coverage model.BackupHistoryCoverage,
	available bool,
	from, now time.Time,
	location *time.Location,
) BackupJobsPayload {
	to := backupJobsObservedThrough(workloads, coverage, now)
	lastFrom := now.Add(-24 * time.Hour)
	days := recovery.BuildBackupHealth(workloads, backupJobPointsSince(points, from), coverage, from, to, location)
	last := recovery.BuildBackupHealth(workloads, backupJobPointsSince(points, lastFrom), coverage, lastFrom, to, location)

	payload := BackupJobsPayload{
		HistoryState: days.HistoryState,
		Timezone:     location.String(),
		Last24h: BackupJobsWindow{
			Start:        last.Start,
			End:          last.End,
			HistoryState: last.HistoryState,
			Counts:       backupJobsCounts(last.Counts),
			Failures:     make([]BackupJobsFailure, 0, len(last.Failures)),
		},
		Days: make([]BackupJobsDay, 0, len(days.Days)),
	}
	for _, failure := range last.Failures {
		payload.Last24h.Failures = append(payload.Last24h.Failures, BackupJobsFailure{
			WorkloadID:   failure.WorkloadID,
			WorkloadName: failure.WorkloadName,
			Instance:     failure.Instance,
			At:           failure.At,
			Outcome:      string(failure.Outcome),
		})
	}
	for _, day := range days.Days {
		payload.Days = append(payload.Days, BackupJobsDay{
			Date:         day.Date,
			HistoryState: day.HistoryState,
			Counts:       backupJobsCounts(day.Counts),
		})
	}
	if !available {
		// No successful backups are assumed when history cannot be read.
		payload.HistoryState = "unavailable"
		payload.Last24h.HistoryState = "unavailable"
		for i := range payload.Days {
			payload.Days[i].HistoryState = "unavailable"
		}
	}

	return payload
}

// pveBackupJobWorkloads keeps Proxmox VE guests only. A VM from another
// platform has no PVE instance; BuildBackupHealth would rate its empty
// instance as an unobserved source and mark every day partial.
func pveBackupJobWorkloads(workloads []model.BackupHealthWorkload) []model.BackupHealthWorkload {
	out := make([]model.BackupHealthWorkload, 0, len(workloads))
	for _, workload := range workloads {
		if workload.Instance != "" && workload.VMID > 0 {
			out = append(out, workload)
		}
	}
	return out
}

// Collection is recorded per completed poll, so coverage always ends a poll
// interval before the request. A window that ran to "now" would read partial
// on every healthy install. When every rated instance was observed within
// backupJobsCollectionGrace, the windows end at the oldest of those latest
// observations instead; anything staler keeps "now" and reads partial.
const backupJobsCollectionGrace = 30 * time.Minute

func backupJobsObservedThrough(workloads []model.BackupHealthWorkload, coverage model.BackupHistoryCoverage, now time.Time) time.Time {
	latest := map[string]time.Time{}
	for _, interval := range coverage.Intervals {
		if interval.Provider != recovery.ProviderProxmoxPVE {
			continue
		}
		if interval.End.After(latest[interval.Instance]) {
			latest[interval.Instance] = interval.End
		}
	}
	through := now
	for _, workload := range workloads {
		end, ok := latest[workload.Instance]
		if !ok || now.Sub(end) > backupJobsCollectionGrace {
			return now
		}
		if end.Before(through) {
			through = end
		}
	}
	return through
}

// backupJobPointsSince keeps tasks that started or finished inside a window.
// BuildBackupHealth clamps an unresolved older task to the window start, so
// one abandoned task would otherwise read "unknown" in every window.
func backupJobPointsSince(points []recovery.RecoveryPoint, start time.Time) []recovery.RecoveryPoint {
	out := make([]recovery.RecoveryPoint, 0, len(points))
	for _, point := range points {
		started := point.StartedAt != nil && !point.StartedAt.Before(start)
		finished := point.CompletedAt != nil && !point.CompletedAt.Before(start)
		if started || finished {
			out = append(out, point)
		}
	}
	return out
}

func backupJobsCounts(counts model.BackupOutcomeCounts) BackupJobsCounts {
	return BackupJobsCounts{
		Success: counts.Success,
		Failed:  counts.Failed,
		Warning: counts.Warning,
		Running: counts.Running,
		Unknown: counts.Unknown,
	}
}

// Mock mode has no retained store. Its fixture tasks are the whole history,
// so map them as the monitor would and treat every rated instance as observed
// for the full window, as a polled real instance would be.
func mockBackupJobHistory(tasks []models.BackupTask, workloads []model.BackupHealthWorkload, from, to time.Time) ([]recovery.RecoveryPoint, model.BackupHistoryCoverage) {
	points, err := proxmoxrecoverymapper.FromPVEBackupTasksWithEvidence(tasks, nil, to)
	if err != nil {
		return nil, model.BackupHistoryCoverage{}
	}
	coverage := model.BackupHistoryCoverage{}
	seen := map[string]bool{}
	instances := make([]string, 0, len(tasks)+len(workloads))
	for _, task := range tasks {
		instances = append(instances, task.Instance)
	}
	for _, workload := range workloads {
		instances = append(instances, workload.Instance)
	}
	for _, instance := range instances {
		if instance == "" || seen[instance] {
			continue
		}
		seen[instance] = true
		coverage.Sources = append(coverage.Sources, model.BackupHistorySource{
			Provider: recovery.ProviderProxmoxPVE, Instance: instance,
			FirstObservedAt: from, LastObservedAt: to, Complete: true,
		})
		coverage.Intervals = append(coverage.Intervals, model.BackupHistoryInterval{
			Provider: recovery.ProviderProxmoxPVE, Instance: instance, Start: from, End: to,
		})
	}
	return points, coverage
}

func (r *Router) registerBackupJobsRoutes() {
	r.mux.HandleFunc("/api/backups/jobs", RequireAuth(r.config, RequireScope(config.ScopeMonitoringRead, r.handleBackupJobs)))
}