Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Read PBS job run status from the admin endpoints
rcourtmane7f6c665201e
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex 3057cdde2a..94ae194f2b 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -120,6 +120,22 @@ admission, cancellation, runtime/client replacement and old-worker completion.Source-bound desktop/narrow Backups proof consumes produced partial/recoveryframes. This is truthful collection, not original frozen-guest recovery.+### PBS job run status comes from the admin endpoints (2 November 2026)++`pkg/pbs.Client.GetJobHealthEvidence` reads configured sync, verify and prune+jobs from `/admin/sync`, `/admin/verify` and `/admin/prune`. Those return each+job with `last-run-state`, `last-run-upid`, `last-run-endtime` and+`next-run`; `/config/{family}` returns the same jobs with none of them, so+reading it made every successful job report `no-run-observed` and the Backups+servers table show "Prune not run" or "Verify not run". `/config/{family}` is+read only when `/admin/{family}` is not found; a permission or other error on+`/admin` stays partial evidence for that path rather than silently degrading+to schedule-only facts. Garbage collection already read+`/admin/datastore/{store}/gc`. Against a real PBS 3 the change took eight+configured prune and verify jobs from config-only with no run state to+`last-run-state` OK matched to their tasks; `TestClient_GetJobHealthEvidence_ReadsRunStatusFromAdminEndpoints`+fails against the old reads.+### Docker image replacements invalidate older tag comparisonsSuccessful tag/platform results are cached for six hours. If an unseen localdiff --git a/pkg/pbs/client.go b/pkg/pbs/client.goindex 8017e77b79..677eb2e74b 100644--- a/pkg/pbs/client.go+++ b/pkg/pbs/client.go@@ -1300,9 +1300,15 @@ type JobHealthEvidence struct {Error string `json:"error,omitempty"`}-// GetJobHealthEvidence collects PBS job health facts from job configuration-// and bounded task history. Permission failures are represented as+// GetJobHealthEvidence collects PBS job health facts from job status and+// bounded task history. Permission failures are represented as// partial-permission evidence instead of failing the whole ledger.+//+// Sync, verify and prune jobs are read from /admin/{family}, which returns each+// configured job with its last-run-state, last-run-upid, last-run-endtime and+// next-run. /config/{family} returns the same jobs without any of those fields,+// so reading it alone left every job with no observed run. /config is read+// only when a PBS has no /admin endpoint for the family.func (c *Client) GetJobHealthEvidence(ctx context.Context, datastores []string, opts JobHealthOptions) ([]JobHealthEvidence, error) {taskHistory, taskPartials, tasksErr := c.listTaskHistory(ctx, datastores, opts)taskByUPID := make(map[string]JobHealthEvidence, len(taskHistory))@@ -1314,11 +1320,16 @@ func (c *Client) GetJobHealthEvidence(ctx context.Context, datastores []string,evidence := make([]JobHealthEvidence, 0)matchedTasks := make(map[string]struct{})- collectConfigs := func(family string, enabled bool, path string) {+ collectConfigs := func(family string, enabled bool) {if !enabled {return}+ path := "/admin/" + familyconfigs, err := c.listJobConfigMaps(ctx, path)+ if err != nil && isPBSNotFoundError(err) {+ path = "/config/" + family+ configs, err = c.listJobConfigMaps(ctx, path)+ }if err != nil {if isPBSNotFoundError(err) {return@@ -1349,9 +1360,9 @@ func (c *Client) GetJobHealthEvidence(ctx context.Context, datastores []string,}}- collectConfigs("sync", opts.MonitorSyncJobs, "/config/sync")- collectConfigs("verify", opts.MonitorVerifyJobs, "/config/verify")- collectConfigs("prune", opts.MonitorPruneJobs, "/config/prune")+ collectConfigs("sync", opts.MonitorSyncJobs)+ collectConfigs("verify", opts.MonitorVerifyJobs)+ collectConfigs("prune", opts.MonitorPruneJobs)if opts.MonitorGarbageJobs {for _, datastore := range datastores {diff --git a/pkg/pbs/client_http_test.go b/pkg/pbs/client_http_test.goindex 17f7cd26c9..729b8c0b1e 100644--- a/pkg/pbs/client_http_test.go+++ b/pkg/pbs/client_http_test.go@@ -272,7 +272,7 @@ func TestClient_GetJobHealthEvidence_MergesConfigAndTaskFacts(t *testing.T) {"endtime": 1700000100,},}})- case "/api2/json/config/sync":+ case "/api2/json/admin/sync":_ = json.NewEncoder(w).Encode(map[string]any{"data": []map[string]any{{"id": "sync-remote-a","store": "fast",@@ -283,13 +283,13 @@ func TestClient_GetJobHealthEvidence_MergesConfigAndTaskFacts(t *testing.T) {"last-run-endtime": 1700000060,"next-run": 1700003600,}}})- case "/api2/json/config/verify":+ case "/api2/json/admin/verify":_ = json.NewEncoder(w).Encode(map[string]any{"data": []map[string]any{{"id": "verify-fast","store": "fast","last-run-state": "OK",}}})- case "/api2/json/config/prune":+ case "/api2/json/admin/prune":w.WriteHeader(http.StatusForbidden)_, _ = w.Write([]byte("permission denied"))case "/api2/json/admin/datastore/fast/gc":@@ -341,6 +341,87 @@ func TestClient_GetJobHealthEvidence_MergesConfigAndTaskFacts(t *testing.T) {}}+// A real PBS returns run results only from /admin/{family}; /config/{family}+// lists the same jobs with schedules and retention but no last-run fields.+// Reading /config made every successful prune, verify and sync job look as if+// it had never run.+func TestClient_GetJobHealthEvidence_ReadsRunStatusFromAdminEndpoints(t *testing.T) {+ var configReads []string+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ switch r.URL.Path {+ case "/api2/json/nodes/localhost/tasks":+ _ = json.NewEncoder(w).Encode(map[string]any{"data": []map[string]any{}})+ case "/api2/json/admin/prune":+ _ = json.NewEncoder(w).Encode(map[string]any{"data": []map[string]any{{+ "id": "prune-main-delly",+ "store": "main",+ "ns": "delly",+ "schedule": "07:10",+ "keep-daily": 3,+ "last-run-state": "OK",+ "last-run-upid": "UPID:pbs:prunejob:main:delly",+ "last-run-endtime": 1793603400,+ "next-run": 1793689800,+ }}})+ case "/api2/json/admin/verify":+ _ = json.NewEncoder(w).Encode(map[string]any{"data": []map[string]any{{+ "id": "verify-main",+ "store": "main",+ "schedule": "sun 05:00",+ "last-run-state": "OK",+ "last-run-endtime": 1793509200,+ "next-run": 1794114000,+ }}})+ case "/api2/json/admin/sync":+ // An older PBS without the status endpoint falls back to /config.+ w.WriteHeader(http.StatusNotFound)+ _, _ = w.Write([]byte("Path '/api2/json/admin/sync' not found."))+ case "/api2/json/config/sync":+ configReads = append(configReads, r.URL.Path)+ _ = json.NewEncoder(w).Encode(map[string]any{"data": []map[string]any{{+ "id": "sync-offsite",+ "store": "main",+ "schedule": "hourly",+ }}})+ case "/api2/json/config/prune", "/api2/json/config/verify":+ configReads = append(configReads, r.URL.Path)+ _ = json.NewEncoder(w).Encode(map[string]any{"data": []map[string]any{}})+ default:+ t.Fatalf("unexpected path: %s", r.URL.Path)+ }+ }))+ defer server.Close()++ client, err := NewClient(ClientConfig{Host: server.URL, TokenName: "root@pam!token", TokenValue: "secret"})+ if err != nil {+ t.Fatalf("NewClient: %v", err)+ }+ evidence, err := client.GetJobHealthEvidence(context.Background(), []string{"main"}, JobHealthOptions{+ MonitorSyncJobs: true,+ MonitorVerifyJobs: true,+ MonitorPruneJobs: true,+ })+ if err != nil {+ t.Fatalf("GetJobHealthEvidence: %v", err)+ }+ byID := make(map[string]JobHealthEvidence)+ for _, item := range evidence {+ byID[item.ID] = item+ }+ if got := byID["prune-main-delly"]; got.LastRunState != "OK" || got.LastRunEndtime != 1793603400 || got.NextRun != 1793689800 || got.Confidence != JobEvidenceConfidenceConfigLastRun {+ t.Fatalf("expected prune run status from /admin/prune, got %+v", got)+ }+ if got := byID["verify-main"]; got.LastRunState != "OK" || got.Confidence != JobEvidenceConfidenceConfigLastRun {+ t.Fatalf("expected verify run status from /admin/verify, got %+v", got)+ }+ if got := byID["sync-offsite"]; got.Schedule != "hourly" || got.LastRunState != "" || got.Confidence != JobEvidenceConfidenceConfigOnly {+ t.Fatalf("expected config-only sync evidence after /admin/sync was not found, got %+v", got)+ }+ if len(configReads) != 1 || configReads[0] != "/api2/json/config/sync" {+ t.Fatalf("expected /config to be read only for the family without /admin, got %v", configReads)+ }+}+func TestPBSJobTaskHistoryQueries_UsesWorkerTypeFilters(t *testing.T) {queries := buildTaskHistoryQueries([]string{"fast", "fast", "slow"}, JobHealthOptions{MonitorBackups: true,