Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Read PBS job status the way Proxmox does
rcourtman2bafa37d5b7c
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex d43897b90f..b7199d048b 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -154,7 +154,12 @@ failure clears on an observed success or warning, or when a complete readdrops or disables the job. Jobs are keyed by family plus id, since PBS ids areunique only within a family. `pbsJobHealthPosture` now ranks a failed lastrun above overdue, so a failed job that has also missed its next run reads-failed, not late. This keeps the datastore job alert from flapping (#1815).+failed, not late. `pbsTaskStatusOutcome` reads a PBS status the way+Proxmox's own UI does (`parse_task_status`): exactly "OK" is success, a+"WARNINGS:" prefix is a warning, "unknown" is unknown, empty is no observed+run, and any other text is a failure, because PBS records a failed run as+the bare error message. Substring matching had called "broken pipe" a+success and left failures without the words "fail" or "error" unknown. This keeps the datastore job alert from flapping (#1815).Mock PBS instances report job results with one failed secondary sync.### PBS job run status comes from the admin endpoints (2 November 2026)diff --git a/internal/monitoring/monitor_pbs_coverage_test.go b/internal/monitoring/monitor_pbs_coverage_test.goindex 5b1db601ef..aededbc438 100644--- a/internal/monitoring/monitor_pbs_coverage_test.go+++ b/internal/monitoring/monitor_pbs_coverage_test.go@@ -950,3 +950,31 @@ func TestPBSJobHealthPostureRanksAFailedRunAboveOverdue(t *testing.T) {t.Fatalf("freshness/posture = %s %s/%s, want overdue critical/last-run-failed", got.Freshness.State, got.Posture, got.PostureReason)}}++// PBS reports a failed run as the bare error message. Every message that is+// not OK, a WARNINGS count or unknown is a failure, whatever words it holds.+func TestPBSJobHealthPostureReadsStatusLikeProxmox(t *testing.T) {+ observedAt := time.Unix(1700010000, 0)+ cases := []struct {+ status, posture, reason string+ }{+ {"OK", "healthy", "last-run-success"},+ {"ok", "healthy", "last-run-success"},+ {"WARNINGS: 2", "warning", "last-run-warning"},+ {"unknown", "unknown", "last-run-state-unknown"},+ {"", "unknown", "no-run-observed"},+ {"remote 'offsite' does not exist", "critical", "last-run-failed"},+ {"broken pipe", "critical", "last-run-failed"},+ {"token lookup timed out", "critical", "last-run-failed"},+ {"connection error", "critical", "last-run-failed"},+ }+ for _, tc := range cases {+ evidence := pbsJobHealthEvidenceFromFacts([]pbs.JobHealthEvidence{{+ ID: "s-1", Family: "sync", Store: "offsite", Enabled: true,+ LastRunState: tc.status, LastRunEndtime: 1700009000,+ }}, observedAt)+ if got := evidence[0]; got.Posture != tc.posture || got.PostureReason != tc.reason {+ t.Fatalf("status %q: posture = %s/%s, want %s/%s", tc.status, got.Posture, got.PostureReason, tc.posture, tc.reason)+ }+ }+}diff --git a/internal/monitoring/monitor_pbs_pmg.go b/internal/monitoring/monitor_pbs_pmg.goindex 0a1075c912..332a95cdbe 100644--- a/internal/monitoring/monitor_pbs_pmg.go+++ b/internal/monitoring/monitor_pbs_pmg.go@@ -147,31 +147,53 @@ func pbsJobHealthFreshness(fact pbs.JobHealthEvidence, observedAt time.Time) mod}func pbsJobHealthPosture(fact pbs.JobHealthEvidence, freshness models.PBSJobHealthFreshness) (string, string) {- status := strings.ToLower(strings.TrimSpace(firstNonEmptyString(fact.TaskStatus, fact.LastRunState)))if fact.Error != "" {return "unknown", strings.TrimSpace(fact.Confidence)}+ outcome := pbsTaskStatusOutcome(firstNonEmptyString(fact.TaskStatus, fact.LastRunState))// A failed last run outranks a missed next one: an overdue job whose last// run failed is still failed, not merely late.- if strings.Contains(status, "fail") || strings.Contains(status, "error") {+ if outcome == "failed" {return "critical", "last-run-failed"}if freshness.State == "overdue" {return "warning", "job-overdue"}- switch {- case status == "":+ switch outcome {+ case "":return "unknown", "no-run-observed"- case strings.Contains(status, "ok") || strings.Contains(status, "success"):+ case "ok":return "healthy", "last-run-success"- case strings.Contains(status, "running"):+ case "running":return "healthy", "running"- case strings.Contains(status, "warn"):+ case "warning":return "warning", "last-run-warning"- case strings.Contains(status, "fail") || strings.Contains(status, "error"):- return "critical", "last-run-failed"default:- return "unknown", "last-run-state-" + strings.ReplaceAll(status, " ", "-")+ return "unknown", "last-run-state-unknown"+ }+}++// pbsTaskStatusOutcome reads a PBS task or job status the way Proxmox's own UI+// does (proxmox-widget-toolkit parse_task_status). PBS writes "OK",+// "WARNINGS: <n>", "unknown", or, for a failure, the error message itself with+// no prefix, such as "remote 'offsite' does not exist" or "broken pipe".+// Matching words like "error" missed failures that lack them, and matching+// "ok" called "broken pipe" a success.+func pbsTaskStatusOutcome(status string) string {+ lower := strings.ToLower(strings.TrimSpace(status))+ switch {+ case lower == "":+ return ""+ case lower == "ok":+ return "ok"+ case strings.HasPrefix(lower, "warnings:"):+ return "warning"+ case lower == "unknown":+ return "unknown"+ case lower == "running":+ return "running"+ default:+ return "failed"}}