Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Keep PBS job alerts through outages and partial reads
rcourtmand87938bb2479
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex b7199d048b..3e4e6deacf 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -152,7 +152,14 @@ drops the previous result while the next run is in progress, an overdue orunreadable job reports none, and a failed read lists no jobs. The carriedfailure clears on an observed success or warning, or when a complete readdrops or disables the job. Jobs are keyed by family plus id, since PBS ids are-unique only within a family. `pbsJobHealthPosture` now ranks a failed last+unique only within a family. Only a failed job-list read (the `<family>:partial`+entry) makes a family unreadable; task-history trouble does not keep a+removed job's failure. Recovery is judged from the run's own status, so a+successful run first seen after its next schedule passed still clears the+carry. A poll that fails before reading jobs keeps the instance's last job+results in state, so the carry survives a PBS outage. Carried failures live+in memory: a Pulse restart forgets them until PBS reports the job again.+`pbsJobHealthPosture` now ranks a failed lastrun above overdue, so a failed job that has also missed its next run readsfailed, not late. `pbsTaskStatusOutcome` reads a PBS status the wayProxmox's own UI does (`parse_task_status`): exactly "OK" is success, adiff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.mdindex cc508c9d37..f94d93b84f 100644--- a/docs/release-control/v6/internal/subsystems/unified-resources.md+++ b/docs/release-control/v6/internal/subsystems/unified-resources.md@@ -231,8 +231,11 @@ recoverability, the native id is `pbs-datastore:<store>:job:<family>:<job id>`(PBS job ids are unique only within a family), and thesummary names the job and a sync's remote without times, because incidentsmerge on their summary. The existing unified-incident pipeline raises a-`backup-storage-incident` alert on the datastore and clears it when the job's-newest run succeeds; the storage disable switch and per-resource overrides+`backup-storage-incident` alert on the datastore and clears it after two+polls that see the job's newest run succeed (`RecoveryConfirmationsRequired`+2). A PBS outage drops the datastore from a poll, which with confirmed+recovery reads as unknown, so the alert neither clears nor notifies again+when PBS returns. The storage disable switch and per-resource overridesapply. Storeless task history, disabled jobs, backup-family jobs and warning,late or unknown runs raise nothing. So the alert neither flaps nor clearsearly, the PBS poller carries a job's last completed failurediff --git a/internal/alerts/unified_incidents_test.go b/internal/alerts/unified_incidents_test.goindex 7e1443f327..c774bdda9f 100644--- a/internal/alerts/unified_incidents_test.go+++ b/internal/alerts/unified_incidents_test.go@@ -1466,8 +1466,19 @@ func TestSyncUnifiedResourceIncidentsAlertsOnFailedPBSSyncJob(t *testing.T) {t.Fatalf("resource name = %q, want the sync's store", alert.ResourceName)}+ // A PBS outage drops the datastore from the poll: unknown, not recovery.+ m.SyncUnifiedResourceIncidents(nil)+ if active := m.GetActiveAlerts(); len(active) != 1 {+ t.Fatalf("an outage must keep the alert, got %+v", active)+ }++ // Recovery needs two polls that see the job succeed.+ m.SyncUnifiedResourceIncidents(snapshot("healthy"))+ if active := m.GetActiveAlerts(); len(active) != 1 {+ t.Fatalf("one successful poll must not clear the alert yet, got %+v", active)+ }m.SyncUnifiedResourceIncidents(snapshot("healthy"))if active := m.GetActiveAlerts(); len(active) != 0 {- t.Fatalf("a succeeded sync must clear the alert, got %+v", active)+ t.Fatalf("a confirmed successful sync must clear the alert, got %+v", active)}}diff --git a/internal/monitoring/monitor_pbs_coverage_test.go b/internal/monitoring/monitor_pbs_coverage_test.goindex aededbc438..ce20b43388 100644--- a/internal/monitoring/monitor_pbs_coverage_test.go+++ b/internal/monitoring/monitor_pbs_coverage_test.go@@ -863,12 +863,13 @@ func TestPBSBudgetOwnerWithdrawsLatePublication(t *testing.T) {exercisePBSBudgetOwnerWithdrawal(t)}-func carriedJob(id, family, posture, reason string) models.PBSJobHealthEvidence {+func carriedJob(id, family, state, posture, reason string) models.PBSJobHealthEvidence {return models.PBSJobHealthEvidence{ID: id,Family: family,Store: "offsite",Enabled: true,+ LastRunState: state,Posture: posture,PostureReason: reason,}@@ -886,7 +887,7 @@ func postureOf(evidence []models.PBSJobHealthEvidence, family, id string) string// A failed sync stays failed through the polls that cannot report a newer// outcome (#1815): otherwise its alert would clear and fire again each time.func TestCarryPBSJobFailuresHoldsAFailureUntilAnOutcomeIsObserved(t *testing.T) {- failed := []models.PBSJobHealthEvidence{carriedJob("s-1", "sync", "critical", "last-run-failed")}+ failed := []models.PBSJobHealthEvidence{carriedJob("s-1", "sync", "connection error", "critical", "last-run-failed")}cases := []struct {name string@@ -894,14 +895,19 @@ func TestCarryPBSJobFailuresHoldsAFailureUntilAnOutcomeIsObserved(t *testing.T)readFailed boolwant string}{- {"next run in progress", []models.PBSJobHealthEvidence{carriedJob("s-1", "sync", "healthy", "running")}, false, "critical/last-run-failed"},- {"overdue with no newer run", []models.PBSJobHealthEvidence{carriedJob("s-1", "sync", "warning", "job-overdue")}, false, "critical/last-run-failed"},- {"state unreadable", []models.PBSJobHealthEvidence{carriedJob("s-1", "sync", "unknown", "no-run-observed")}, false, "critical/last-run-failed"},+ // PBS drops the previous result while the next run is in progress.+ {"next run in progress", []models.PBSJobHealthEvidence{carriedJob("s-1", "sync", "", "unknown", "no-run-observed")}, false, "critical/last-run-failed"},+ {"state unknown", []models.PBSJobHealthEvidence{carriedJob("s-1", "sync", "unknown", "unknown", "last-run-state-unknown")}, false, "critical/last-run-failed"},{"whole read failed", nil, true, "critical/last-run-failed"},- {"family read failed", []models.PBSJobHealthEvidence{{ID: "sync:partial", Family: "sync", Error: "403"}}, false, "critical/last-run-failed"},- {"retry succeeded", []models.PBSJobHealthEvidence{carriedJob("s-1", "sync", "healthy", "last-run-success")}, false, "healthy/last-run-success"},- {"retry warned", []models.PBSJobHealthEvidence{carriedJob("s-1", "sync", "warning", "last-run-warning")}, false, "warning/last-run-warning"},+ {"job list read failed", []models.PBSJobHealthEvidence{{ID: "sync:partial", Family: "sync", Error: "403", EvidenceSource: pbs.JobEvidenceSourcePBSPartialRead}}, false, "critical/last-run-failed"},+ {"retry succeeded", []models.PBSJobHealthEvidence{carriedJob("s-1", "sync", "OK", "healthy", "last-run-success")}, false, "healthy/last-run-success"},+ {"retry warned", []models.PBSJobHealthEvidence{carriedJob("s-1", "sync", "WARNINGS: 1", "warning", "last-run-warning")}, false, "warning/last-run-warning"},+ // A successful run first seen after the next schedule passed is still+ // a recovery, even though the posture reads late.+ {"retry succeeded, now overdue", []models.PBSJobHealthEvidence{carriedJob("s-1", "sync", "OK", "warning", "job-overdue")}, false, "warning/job-overdue"},{"job removed", []models.PBSJobHealthEvidence{}, false, "absent"},+ // Task history trouble says nothing about the configured job list.+ {"job removed while task history failed", []models.PBSJobHealthEvidence{{ID: "sync:task-history-partial", Family: "sync", Error: "500", EvidenceSource: pbs.JobEvidenceSourcePBSPartialRead}}, false, "absent"},}for _, tc := range cases {t.Run(tc.name, func(t *testing.T) {@@ -912,7 +918,7 @@ func TestCarryPBSJobFailuresHoldsAFailureUntilAnOutcomeIsObserved(t *testing.T)})}- disabled := carriedJob("s-1", "sync", "unknown", "no-run-observed")+ disabled := carriedJob("s-1", "sync", "", "unknown", "no-run-observed")disabled.Enabled = falsegot := carryPBSJobFailures(failed, []models.PBSJobHealthEvidence{disabled}, false)if len(got) != 1 || got[0].Enabled {@@ -921,10 +927,10 @@ func TestCarryPBSJobFailuresHoldsAFailureUntilAnOutcomeIsObserved(t *testing.T)}func TestCarryPBSJobFailuresKeysJobsByFamily(t *testing.T) {- previous := []models.PBSJobHealthEvidence{carriedJob("daily", "sync", "critical", "last-run-failed")}+ previous := []models.PBSJobHealthEvidence{carriedJob("daily", "sync", "connection error", "critical", "last-run-failed")}current := []models.PBSJobHealthEvidence{- carriedJob("daily", "verify", "healthy", "last-run-success"),- carriedJob("daily", "sync", "healthy", "running"),+ carriedJob("daily", "verify", "OK", "healthy", "last-run-success"),+ carriedJob("daily", "sync", "", "unknown", "no-run-observed"),}got := carryPBSJobFailures(previous, current, false)if posture := postureOf(got, "sync", "daily"); posture != "critical/last-run-failed" {diff --git a/internal/monitoring/monitor_pbs_pmg.go b/internal/monitoring/monitor_pbs_pmg.goindex 332a95cdbe..dd6d9bf0fc 100644--- a/internal/monitoring/monitor_pbs_pmg.go+++ b/internal/monitoring/monitor_pbs_pmg.go@@ -224,8 +224,10 @@ func carryPBSJobFailures(previous, current []models.PBSJobHealthEvidence, readFaunreadFamilies := make(map[string]bool)for _, entry := range current {- if strings.TrimSpace(entry.Error) != "" {- unreadFamilies[strings.ToLower(strings.TrimSpace(entry.Family))] = true+ family := strings.ToLower(strings.TrimSpace(entry.Family))+ if strings.TrimSpace(entry.Error) != "" && entry.ID == family+":partial" &&+ entry.EvidenceSource == pbs.JobEvidenceSourcePBSPartialRead {+ unreadFamilies[family] = true}}@@ -252,11 +254,15 @@ func carryPBSJobFailures(previous, current []models.PBSJobHealthEvidence, readFa}// pbsJobReportsOutcome says whether this poll observed how the job's newest-// run ended: success, warning or failure, rather than running, overdue or an-// unreadable state.+// run ended: success, warning or failure, rather than running or an+// unreadable state. It reads the run's own status, not the posture, so a+// successful run first seen after its next schedule passed still counts.func pbsJobReportsOutcome(entry models.PBSJobHealthEvidence) bool {- switch entry.PostureReason {- case "last-run-success", "last-run-warning", "last-run-failed":+ if strings.TrimSpace(entry.Error) != "" {+ return false+ }+ switch pbsTaskStatusOutcome(firstNonEmptyString(entry.TaskStatus, entry.LastRunState)) {+ case "ok", "warning", "failed":return truedefault:return false@@ -743,6 +749,11 @@ func (m *Monitor) recordPBSNodeHistory(serviceID string, status *pbs.NodeStatus,// connection map, dashboard resource, and legacy alert evaluator must all use// the same poll error that is recorded in the scheduler ledger.func (m *Monitor) publishPBSConnectionOutcome(instance models.PBSInstance, pollErr error) {+ // A poll that failed before reading jobs knows nothing new about them;+ // keep the last results so a failed job is still carried when PBS returns.+ if pollErr != nil && len(instance.JobHealthEvidence) == 0 {+ instance.JobHealthEvidence = m.state.PBSJobHealthEvidence(instance.ID)+ }connected := pollErr == nilif connected {instance.Status = "online"diff --git a/internal/unifiedresources/pbs_job_incidents.go b/internal/unifiedresources/pbs_job_incidents.goindex d9c1983653..7f848809a1 100644--- a/internal/unifiedresources/pbs_job_incidents.go+++ b/internal/unifiedresources/pbs_job_incidents.go@@ -89,6 +89,10 @@ func pbsDatastoreJobIncidents(instance models.PBSInstance, datastore string) []RSource: string(SourcePBS),Summary: pbsJobFailedSummary(entry, id, store),StartedAt: startedAt,+ // A PBS outage drops the datastore from a poll; with confirmed+ // recovery that reads as unknown, so the alert neither clears nor+ // notifies again when PBS returns.+ RecoveryConfirmationsRequired: 2,})}return incidentsdiff --git a/internal/unifiedresources/pbs_pmg_registry_test.go b/internal/unifiedresources/pbs_pmg_registry_test.goindex 4bdbcea98f..c89fce4b0d 100644--- a/internal/unifiedresources/pbs_pmg_registry_test.go+++ b/internal/unifiedresources/pbs_pmg_registry_test.go@@ -932,6 +932,11 @@ func TestPBSDatastoreJobIncidentsNameGarbageCollectionByStore(t *testing.T) {if got[0].Summary != "Garbage collection failed on datastore main" || got[1].Summary != "Prune job p-1 failed on datastore main" {t.Fatalf("summaries = %q, %q", got[0].Summary, got[1].Summary)}+ // A PBS outage drops the datastore from a poll; confirmed recovery keeps+ // that from reading as the job recovering.+ if got[0].RecoveryConfirmationsRequired != 2 {+ t.Fatalf("recovery confirmations = %d, want 2", got[0].RecoveryConfirmationsRequired)+ }}// PBS job ids are unique only within a family; a later successful verify job