Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Read PBS job status the way Proxmox does

rcourtman2bafa37d5b7c
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index d43897b90f..b7199d048b 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -154,7 +154,12 @@ failure clears on an observed success or warning, or when a complete read
drops or disables the job. Jobs are keyed by family plus id, since PBS ids are
unique only within a family. `pbsJobHealthPosture` now ranks a failed last
run above overdue, so a failed job that has also missed its next run reads
-failed, not late. This keeps the datastore job alert from flapping (#1815).
+failed, not late. `pbsTaskStatusOutcome` reads a PBS status the way
+Proxmox's own UI does (`parse_task_status`): exactly "OK" is success, a
+"WARNINGS:" prefix is a warning, "unknown" is unknown, empty is no observed
+run, and any other text is a failure, because PBS records a failed run as
+the bare error message. Substring matching had called "broken pipe" a
+success and left failures without the words "fail" or "error" unknown. This keeps the datastore job alert from flapping (#1815).
Mock PBS instances report job results with one failed secondary sync.
### PBS job run status comes from the admin endpoints (2 November 2026)
diff --git a/internal/monitoring/monitor_pbs_coverage_test.go b/internal/monitoring/monitor_pbs_coverage_test.go
index 5b1db601ef..aededbc438 100644
--- a/internal/monitoring/monitor_pbs_coverage_test.go
+++ b/internal/monitoring/monitor_pbs_coverage_test.go
@@ -950,3 +950,31 @@ func TestPBSJobHealthPostureRanksAFailedRunAboveOverdue(t *testing.T) {
t.Fatalf("freshness/posture = %s %s/%s, want overdue critical/last-run-failed", got.Freshness.State, got.Posture, got.PostureReason)
}
}
+
+// PBS reports a failed run as the bare error message. Every message that is
+// not OK, a WARNINGS count or unknown is a failure, whatever words it holds.
+func TestPBSJobHealthPostureReadsStatusLikeProxmox(t *testing.T) {
+ observedAt := time.Unix(1700010000, 0)
+ cases := []struct {
+ status, posture, reason string
+ }{
+ {"OK", "healthy", "last-run-success"},
+ {"ok", "healthy", "last-run-success"},
+ {"WARNINGS: 2", "warning", "last-run-warning"},
+ {"unknown", "unknown", "last-run-state-unknown"},
+ {"", "unknown", "no-run-observed"},
+ {"remote 'offsite' does not exist", "critical", "last-run-failed"},
+ {"broken pipe", "critical", "last-run-failed"},
+ {"token lookup timed out", "critical", "last-run-failed"},
+ {"connection error", "critical", "last-run-failed"},
+ }
+ for _, tc := range cases {
+ evidence := pbsJobHealthEvidenceFromFacts([]pbs.JobHealthEvidence{{
+ ID: "s-1", Family: "sync", Store: "offsite", Enabled: true,
+ LastRunState: tc.status, LastRunEndtime: 1700009000,
+ }}, observedAt)
+ if got := evidence[0]; got.Posture != tc.posture || got.PostureReason != tc.reason {
+ t.Fatalf("status %q: posture = %s/%s, want %s/%s", tc.status, got.Posture, got.PostureReason, tc.posture, tc.reason)
+ }
+ }
+}
diff --git a/internal/monitoring/monitor_pbs_pmg.go b/internal/monitoring/monitor_pbs_pmg.go
index 0a1075c912..332a95cdbe 100644
--- a/internal/monitoring/monitor_pbs_pmg.go
+++ b/internal/monitoring/monitor_pbs_pmg.go
@@ -147,31 +147,53 @@ func pbsJobHealthFreshness(fact pbs.JobHealthEvidence, observedAt time.Time) mod
}
func pbsJobHealthPosture(fact pbs.JobHealthEvidence, freshness models.PBSJobHealthFreshness) (string, string) {
- status := strings.ToLower(strings.TrimSpace(firstNonEmptyString(fact.TaskStatus, fact.LastRunState)))
if fact.Error != "" {
return "unknown", strings.TrimSpace(fact.Confidence)
}
+ outcome := pbsTaskStatusOutcome(firstNonEmptyString(fact.TaskStatus, fact.LastRunState))
// A failed last run outranks a missed next one: an overdue job whose last
// run failed is still failed, not merely late.
- if strings.Contains(status, "fail") || strings.Contains(status, "error") {
+ if outcome == "failed" {
return "critical", "last-run-failed"
}
if freshness.State == "overdue" {
return "warning", "job-overdue"
}
- switch {
- case status == "":
+ switch outcome {
+ case "":
return "unknown", "no-run-observed"
- case strings.Contains(status, "ok") || strings.Contains(status, "success"):
+ case "ok":
return "healthy", "last-run-success"
- case strings.Contains(status, "running"):
+ case "running":
return "healthy", "running"
- case strings.Contains(status, "warn"):
+ case "warning":
return "warning", "last-run-warning"
- case strings.Contains(status, "fail") || strings.Contains(status, "error"):
- return "critical", "last-run-failed"
default:
- return "unknown", "last-run-state-" + strings.ReplaceAll(status, " ", "-")
+ return "unknown", "last-run-state-unknown"
+ }
+}
+
+// pbsTaskStatusOutcome reads a PBS task or job status the way Proxmox's own UI
+// does (proxmox-widget-toolkit parse_task_status). PBS writes "OK",
+// "WARNINGS: <n>", "unknown", or, for a failure, the error message itself with
+// no prefix, such as "remote 'offsite' does not exist" or "broken pipe".
+// Matching words like "error" missed failures that lack them, and matching
+// "ok" called "broken pipe" a success.
+func pbsTaskStatusOutcome(status string) string {
+ lower := strings.ToLower(strings.TrimSpace(status))
+ switch {
+ case lower == "":
+ return ""
+ case lower == "ok":
+ return "ok"
+ case strings.HasPrefix(lower, "warnings:"):
+ return "warning"
+ case lower == "unknown":
+ return "unknown"
+ case lower == "running":
+ return "running"
+ default:
+ return "failed"
}
}