Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Keep PBS job alerts through outages and partial reads

rcourtmand87938bb2479
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index b7199d048b..3e4e6deacf 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -152,7 +152,14 @@ drops the previous result while the next run is in progress, an overdue or
unreadable job reports none, and a failed read lists no jobs. The carried
failure clears on an observed success or warning, or when a complete read
drops or disables the job. Jobs are keyed by family plus id, since PBS ids are
-unique only within a family. `pbsJobHealthPosture` now ranks a failed last
+unique only within a family. Only a failed job-list read (the `<family>:partial`
+entry) makes a family unreadable; task-history trouble does not keep a
+removed job's failure. Recovery is judged from the run's own status, so a
+successful run first seen after its next schedule passed still clears the
+carry. A poll that fails before reading jobs keeps the instance's last job
+results in state, so the carry survives a PBS outage. Carried failures live
+in memory: a Pulse restart forgets them until PBS reports the job again.
+`pbsJobHealthPosture` now ranks a failed last
run above overdue, so a failed job that has also missed its next run reads
failed, not late. `pbsTaskStatusOutcome` reads a PBS status the way
Proxmox's own UI does (`parse_task_status`): exactly "OK" is success, a
diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.md
index cc508c9d37..f94d93b84f 100644
--- a/docs/release-control/v6/internal/subsystems/unified-resources.md
+++ b/docs/release-control/v6/internal/subsystems/unified-resources.md
@@ -231,8 +231,11 @@ recoverability, the native id is `pbs-datastore:<store>:job:<family>:<job id>`
(PBS job ids are unique only within a family), and the
summary names the job and a sync's remote without times, because incidents
merge on their summary. The existing unified-incident pipeline raises a
-`backup-storage-incident` alert on the datastore and clears it when the job's
-newest run succeeds; the storage disable switch and per-resource overrides
+`backup-storage-incident` alert on the datastore and clears it after two
+polls that see the job's newest run succeed (`RecoveryConfirmationsRequired`
+2). A PBS outage drops the datastore from a poll, which with confirmed
+recovery reads as unknown, so the alert neither clears nor notifies again
+when PBS returns. The storage disable switch and per-resource overrides
apply. Storeless task history, disabled jobs, backup-family jobs and warning,
late or unknown runs raise nothing. So the alert neither flaps nor clears
early, the PBS poller carries a job's last completed failure
diff --git a/internal/alerts/unified_incidents_test.go b/internal/alerts/unified_incidents_test.go
index 7e1443f327..c774bdda9f 100644
--- a/internal/alerts/unified_incidents_test.go
+++ b/internal/alerts/unified_incidents_test.go
@@ -1466,8 +1466,19 @@ func TestSyncUnifiedResourceIncidentsAlertsOnFailedPBSSyncJob(t *testing.T) {
t.Fatalf("resource name = %q, want the sync's store", alert.ResourceName)
}
+ // A PBS outage drops the datastore from the poll: unknown, not recovery.
+ m.SyncUnifiedResourceIncidents(nil)
+ if active := m.GetActiveAlerts(); len(active) != 1 {
+ t.Fatalf("an outage must keep the alert, got %+v", active)
+ }
+
+ // Recovery needs two polls that see the job succeed.
+ m.SyncUnifiedResourceIncidents(snapshot("healthy"))
+ if active := m.GetActiveAlerts(); len(active) != 1 {
+ t.Fatalf("one successful poll must not clear the alert yet, got %+v", active)
+ }
m.SyncUnifiedResourceIncidents(snapshot("healthy"))
if active := m.GetActiveAlerts(); len(active) != 0 {
- t.Fatalf("a succeeded sync must clear the alert, got %+v", active)
+ t.Fatalf("a confirmed successful sync must clear the alert, got %+v", active)
}
}
diff --git a/internal/monitoring/monitor_pbs_coverage_test.go b/internal/monitoring/monitor_pbs_coverage_test.go
index aededbc438..ce20b43388 100644
--- a/internal/monitoring/monitor_pbs_coverage_test.go
+++ b/internal/monitoring/monitor_pbs_coverage_test.go
@@ -863,12 +863,13 @@ func TestPBSBudgetOwnerWithdrawsLatePublication(t *testing.T) {
exercisePBSBudgetOwnerWithdrawal(t)
}
-func carriedJob(id, family, posture, reason string) models.PBSJobHealthEvidence {
+func carriedJob(id, family, state, posture, reason string) models.PBSJobHealthEvidence {
return models.PBSJobHealthEvidence{
ID: id,
Family: family,
Store: "offsite",
Enabled: true,
+ LastRunState: state,
Posture: posture,
PostureReason: reason,
}
@@ -886,7 +887,7 @@ func postureOf(evidence []models.PBSJobHealthEvidence, family, id string) string
// A failed sync stays failed through the polls that cannot report a newer
// outcome (#1815): otherwise its alert would clear and fire again each time.
func TestCarryPBSJobFailuresHoldsAFailureUntilAnOutcomeIsObserved(t *testing.T) {
- failed := []models.PBSJobHealthEvidence{carriedJob("s-1", "sync", "critical", "last-run-failed")}
+ failed := []models.PBSJobHealthEvidence{carriedJob("s-1", "sync", "connection error", "critical", "last-run-failed")}
cases := []struct {
name string
@@ -894,14 +895,19 @@ func TestCarryPBSJobFailuresHoldsAFailureUntilAnOutcomeIsObserved(t *testing.T)
readFailed bool
want string
}{
- {"next run in progress", []models.PBSJobHealthEvidence{carriedJob("s-1", "sync", "healthy", "running")}, false, "critical/last-run-failed"},
- {"overdue with no newer run", []models.PBSJobHealthEvidence{carriedJob("s-1", "sync", "warning", "job-overdue")}, false, "critical/last-run-failed"},
- {"state unreadable", []models.PBSJobHealthEvidence{carriedJob("s-1", "sync", "unknown", "no-run-observed")}, false, "critical/last-run-failed"},
+ // PBS drops the previous result while the next run is in progress.
+ {"next run in progress", []models.PBSJobHealthEvidence{carriedJob("s-1", "sync", "", "unknown", "no-run-observed")}, false, "critical/last-run-failed"},
+ {"state unknown", []models.PBSJobHealthEvidence{carriedJob("s-1", "sync", "unknown", "unknown", "last-run-state-unknown")}, false, "critical/last-run-failed"},
{"whole read failed", nil, true, "critical/last-run-failed"},
- {"family read failed", []models.PBSJobHealthEvidence{{ID: "sync:partial", Family: "sync", Error: "403"}}, false, "critical/last-run-failed"},
- {"retry succeeded", []models.PBSJobHealthEvidence{carriedJob("s-1", "sync", "healthy", "last-run-success")}, false, "healthy/last-run-success"},
- {"retry warned", []models.PBSJobHealthEvidence{carriedJob("s-1", "sync", "warning", "last-run-warning")}, false, "warning/last-run-warning"},
+ {"job list read failed", []models.PBSJobHealthEvidence{{ID: "sync:partial", Family: "sync", Error: "403", EvidenceSource: pbs.JobEvidenceSourcePBSPartialRead}}, false, "critical/last-run-failed"},
+ {"retry succeeded", []models.PBSJobHealthEvidence{carriedJob("s-1", "sync", "OK", "healthy", "last-run-success")}, false, "healthy/last-run-success"},
+ {"retry warned", []models.PBSJobHealthEvidence{carriedJob("s-1", "sync", "WARNINGS: 1", "warning", "last-run-warning")}, false, "warning/last-run-warning"},
+ // A successful run first seen after the next schedule passed is still
+ // a recovery, even though the posture reads late.
+ {"retry succeeded, now overdue", []models.PBSJobHealthEvidence{carriedJob("s-1", "sync", "OK", "warning", "job-overdue")}, false, "warning/job-overdue"},
{"job removed", []models.PBSJobHealthEvidence{}, false, "absent"},
+ // Task history trouble says nothing about the configured job list.
+ {"job removed while task history failed", []models.PBSJobHealthEvidence{{ID: "sync:task-history-partial", Family: "sync", Error: "500", EvidenceSource: pbs.JobEvidenceSourcePBSPartialRead}}, false, "absent"},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
@@ -912,7 +918,7 @@ func TestCarryPBSJobFailuresHoldsAFailureUntilAnOutcomeIsObserved(t *testing.T)
})
}
- disabled := carriedJob("s-1", "sync", "unknown", "no-run-observed")
+ disabled := carriedJob("s-1", "sync", "", "unknown", "no-run-observed")
disabled.Enabled = false
got := carryPBSJobFailures(failed, []models.PBSJobHealthEvidence{disabled}, false)
if len(got) != 1 || got[0].Enabled {
@@ -921,10 +927,10 @@ func TestCarryPBSJobFailuresHoldsAFailureUntilAnOutcomeIsObserved(t *testing.T)
}
func TestCarryPBSJobFailuresKeysJobsByFamily(t *testing.T) {
- previous := []models.PBSJobHealthEvidence{carriedJob("daily", "sync", "critical", "last-run-failed")}
+ previous := []models.PBSJobHealthEvidence{carriedJob("daily", "sync", "connection error", "critical", "last-run-failed")}
current := []models.PBSJobHealthEvidence{
- carriedJob("daily", "verify", "healthy", "last-run-success"),
- carriedJob("daily", "sync", "healthy", "running"),
+ carriedJob("daily", "verify", "OK", "healthy", "last-run-success"),
+ carriedJob("daily", "sync", "", "unknown", "no-run-observed"),
}
got := carryPBSJobFailures(previous, current, false)
if posture := postureOf(got, "sync", "daily"); posture != "critical/last-run-failed" {
diff --git a/internal/monitoring/monitor_pbs_pmg.go b/internal/monitoring/monitor_pbs_pmg.go
index 332a95cdbe..dd6d9bf0fc 100644
--- a/internal/monitoring/monitor_pbs_pmg.go
+++ b/internal/monitoring/monitor_pbs_pmg.go
@@ -224,8 +224,10 @@ func carryPBSJobFailures(previous, current []models.PBSJobHealthEvidence, readFa
unreadFamilies := make(map[string]bool)
for _, entry := range current {
- if strings.TrimSpace(entry.Error) != "" {
- unreadFamilies[strings.ToLower(strings.TrimSpace(entry.Family))] = true
+ family := strings.ToLower(strings.TrimSpace(entry.Family))
+ if strings.TrimSpace(entry.Error) != "" && entry.ID == family+":partial" &&
+ entry.EvidenceSource == pbs.JobEvidenceSourcePBSPartialRead {
+ unreadFamilies[family] = true
}
}
@@ -252,11 +254,15 @@ func carryPBSJobFailures(previous, current []models.PBSJobHealthEvidence, readFa
}
// pbsJobReportsOutcome says whether this poll observed how the job's newest
-// run ended: success, warning or failure, rather than running, overdue or an
-// unreadable state.
+// run ended: success, warning or failure, rather than running or an
+// unreadable state. It reads the run's own status, not the posture, so a
+// successful run first seen after its next schedule passed still counts.
func pbsJobReportsOutcome(entry models.PBSJobHealthEvidence) bool {
- switch entry.PostureReason {
- case "last-run-success", "last-run-warning", "last-run-failed":
+ if strings.TrimSpace(entry.Error) != "" {
+ return false
+ }
+ switch pbsTaskStatusOutcome(firstNonEmptyString(entry.TaskStatus, entry.LastRunState)) {
+ case "ok", "warning", "failed":
return true
default:
return false
@@ -743,6 +749,11 @@ func (m *Monitor) recordPBSNodeHistory(serviceID string, status *pbs.NodeStatus,
// connection map, dashboard resource, and legacy alert evaluator must all use
// the same poll error that is recorded in the scheduler ledger.
func (m *Monitor) publishPBSConnectionOutcome(instance models.PBSInstance, pollErr error) {
+ // A poll that failed before reading jobs knows nothing new about them;
+ // keep the last results so a failed job is still carried when PBS returns.
+ if pollErr != nil && len(instance.JobHealthEvidence) == 0 {
+ instance.JobHealthEvidence = m.state.PBSJobHealthEvidence(instance.ID)
+ }
connected := pollErr == nil
if connected {
instance.Status = "online"
diff --git a/internal/unifiedresources/pbs_job_incidents.go b/internal/unifiedresources/pbs_job_incidents.go
index d9c1983653..7f848809a1 100644
--- a/internal/unifiedresources/pbs_job_incidents.go
+++ b/internal/unifiedresources/pbs_job_incidents.go
@@ -89,6 +89,10 @@ func pbsDatastoreJobIncidents(instance models.PBSInstance, datastore string) []R
Source: string(SourcePBS),
Summary: pbsJobFailedSummary(entry, id, store),
StartedAt: startedAt,
+ // A PBS outage drops the datastore from a poll; with confirmed
+ // recovery that reads as unknown, so the alert neither clears nor
+ // notifies again when PBS returns.
+ RecoveryConfirmationsRequired: 2,
})
}
return incidents
diff --git a/internal/unifiedresources/pbs_pmg_registry_test.go b/internal/unifiedresources/pbs_pmg_registry_test.go
index 4bdbcea98f..c89fce4b0d 100644
--- a/internal/unifiedresources/pbs_pmg_registry_test.go
+++ b/internal/unifiedresources/pbs_pmg_registry_test.go
@@ -932,6 +932,11 @@ func TestPBSDatastoreJobIncidentsNameGarbageCollectionByStore(t *testing.T) {
if got[0].Summary != "Garbage collection failed on datastore main" || got[1].Summary != "Prune job p-1 failed on datastore main" {
t.Fatalf("summaries = %q, %q", got[0].Summary, got[1].Summary)
}
+ // A PBS outage drops the datastore from a poll; confirmed recovery keeps
+ // that from reading as the job recovering.
+ if got[0].RecoveryConfirmationsRequired != 2 {
+ t.Fatalf("recovery confirmations = %d, want 2", got[0].RecoveryConfirmationsRequired)
+ }
}
// PBS job ids are unique only within a family; a later successful verify job