Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Read PBS job run status from the admin endpoints

rcourtmane7f6c665201e
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index 3057cdde2a..94ae194f2b 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -120,6 +120,22 @@ admission, cancellation, runtime/client replacement and old-worker completion.
Source-bound desktop/narrow Backups proof consumes produced partial/recovery
frames. This is truthful collection, not original frozen-guest recovery.
+### PBS job run status comes from the admin endpoints (2 November 2026)
+
+`pkg/pbs.Client.GetJobHealthEvidence` reads configured sync, verify and prune
+jobs from `/admin/sync`, `/admin/verify` and `/admin/prune`. Those return each
+job with `last-run-state`, `last-run-upid`, `last-run-endtime` and
+`next-run`; `/config/{family}` returns the same jobs with none of them, so
+reading it made every successful job report `no-run-observed` and the Backups
+servers table show "Prune not run" or "Verify not run". `/config/{family}` is
+read only when `/admin/{family}` is not found; a permission or other error on
+`/admin` stays partial evidence for that path rather than silently degrading
+to schedule-only facts. Garbage collection already read
+`/admin/datastore/{store}/gc`. Against a real PBS 3 the change took eight
+configured prune and verify jobs from config-only with no run state to
+`last-run-state` OK matched to their tasks; `TestClient_GetJobHealthEvidence_ReadsRunStatusFromAdminEndpoints`
+fails against the old reads.
+
### Docker image replacements invalidate older tag comparisons
Successful tag/platform results are cached for six hours. If an unseen local
diff --git a/pkg/pbs/client.go b/pkg/pbs/client.go
index 8017e77b79..677eb2e74b 100644
--- a/pkg/pbs/client.go
+++ b/pkg/pbs/client.go
@@ -1300,9 +1300,15 @@ type JobHealthEvidence struct {
Error string `json:"error,omitempty"`
}
-// GetJobHealthEvidence collects PBS job health facts from job configuration
-// and bounded task history. Permission failures are represented as
+// GetJobHealthEvidence collects PBS job health facts from job status and
+// bounded task history. Permission failures are represented as
// partial-permission evidence instead of failing the whole ledger.
+//
+// Sync, verify and prune jobs are read from /admin/{family}, which returns each
+// configured job with its last-run-state, last-run-upid, last-run-endtime and
+// next-run. /config/{family} returns the same jobs without any of those fields,
+// so reading it alone left every job with no observed run. /config is read
+// only when a PBS has no /admin endpoint for the family.
func (c *Client) GetJobHealthEvidence(ctx context.Context, datastores []string, opts JobHealthOptions) ([]JobHealthEvidence, error) {
taskHistory, taskPartials, tasksErr := c.listTaskHistory(ctx, datastores, opts)
taskByUPID := make(map[string]JobHealthEvidence, len(taskHistory))
@@ -1314,11 +1320,16 @@ func (c *Client) GetJobHealthEvidence(ctx context.Context, datastores []string,
evidence := make([]JobHealthEvidence, 0)
matchedTasks := make(map[string]struct{})
- collectConfigs := func(family string, enabled bool, path string) {
+ collectConfigs := func(family string, enabled bool) {
if !enabled {
return
}
+ path := "/admin/" + family
configs, err := c.listJobConfigMaps(ctx, path)
+ if err != nil && isPBSNotFoundError(err) {
+ path = "/config/" + family
+ configs, err = c.listJobConfigMaps(ctx, path)
+ }
if err != nil {
if isPBSNotFoundError(err) {
return
@@ -1349,9 +1360,9 @@ func (c *Client) GetJobHealthEvidence(ctx context.Context, datastores []string,
}
}
- collectConfigs("sync", opts.MonitorSyncJobs, "/config/sync")
- collectConfigs("verify", opts.MonitorVerifyJobs, "/config/verify")
- collectConfigs("prune", opts.MonitorPruneJobs, "/config/prune")
+ collectConfigs("sync", opts.MonitorSyncJobs)
+ collectConfigs("verify", opts.MonitorVerifyJobs)
+ collectConfigs("prune", opts.MonitorPruneJobs)
if opts.MonitorGarbageJobs {
for _, datastore := range datastores {
diff --git a/pkg/pbs/client_http_test.go b/pkg/pbs/client_http_test.go
index 17f7cd26c9..729b8c0b1e 100644
--- a/pkg/pbs/client_http_test.go
+++ b/pkg/pbs/client_http_test.go
@@ -272,7 +272,7 @@ func TestClient_GetJobHealthEvidence_MergesConfigAndTaskFacts(t *testing.T) {
"endtime": 1700000100,
},
}})
- case "/api2/json/config/sync":
+ case "/api2/json/admin/sync":
_ = json.NewEncoder(w).Encode(map[string]any{"data": []map[string]any{{
"id": "sync-remote-a",
"store": "fast",
@@ -283,13 +283,13 @@ func TestClient_GetJobHealthEvidence_MergesConfigAndTaskFacts(t *testing.T) {
"last-run-endtime": 1700000060,
"next-run": 1700003600,
}}})
- case "/api2/json/config/verify":
+ case "/api2/json/admin/verify":
_ = json.NewEncoder(w).Encode(map[string]any{"data": []map[string]any{{
"id": "verify-fast",
"store": "fast",
"last-run-state": "OK",
}}})
- case "/api2/json/config/prune":
+ case "/api2/json/admin/prune":
w.WriteHeader(http.StatusForbidden)
_, _ = w.Write([]byte("permission denied"))
case "/api2/json/admin/datastore/fast/gc":
@@ -341,6 +341,87 @@ func TestClient_GetJobHealthEvidence_MergesConfigAndTaskFacts(t *testing.T) {
}
}
+// A real PBS returns run results only from /admin/{family}; /config/{family}
+// lists the same jobs with schedules and retention but no last-run fields.
+// Reading /config made every successful prune, verify and sync job look as if
+// it had never run.
+func TestClient_GetJobHealthEvidence_ReadsRunStatusFromAdminEndpoints(t *testing.T) {
+ var configReads []string
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch r.URL.Path {
+ case "/api2/json/nodes/localhost/tasks":
+ _ = json.NewEncoder(w).Encode(map[string]any{"data": []map[string]any{}})
+ case "/api2/json/admin/prune":
+ _ = json.NewEncoder(w).Encode(map[string]any{"data": []map[string]any{{
+ "id": "prune-main-delly",
+ "store": "main",
+ "ns": "delly",
+ "schedule": "07:10",
+ "keep-daily": 3,
+ "last-run-state": "OK",
+ "last-run-upid": "UPID:pbs:prunejob:main:delly",
+ "last-run-endtime": 1793603400,
+ "next-run": 1793689800,
+ }}})
+ case "/api2/json/admin/verify":
+ _ = json.NewEncoder(w).Encode(map[string]any{"data": []map[string]any{{
+ "id": "verify-main",
+ "store": "main",
+ "schedule": "sun 05:00",
+ "last-run-state": "OK",
+ "last-run-endtime": 1793509200,
+ "next-run": 1794114000,
+ }}})
+ case "/api2/json/admin/sync":
+ // An older PBS without the status endpoint falls back to /config.
+ w.WriteHeader(http.StatusNotFound)
+ _, _ = w.Write([]byte("Path '/api2/json/admin/sync' not found."))
+ case "/api2/json/config/sync":
+ configReads = append(configReads, r.URL.Path)
+ _ = json.NewEncoder(w).Encode(map[string]any{"data": []map[string]any{{
+ "id": "sync-offsite",
+ "store": "main",
+ "schedule": "hourly",
+ }}})
+ case "/api2/json/config/prune", "/api2/json/config/verify":
+ configReads = append(configReads, r.URL.Path)
+ _ = json.NewEncoder(w).Encode(map[string]any{"data": []map[string]any{}})
+ default:
+ t.Fatalf("unexpected path: %s", r.URL.Path)
+ }
+ }))
+ defer server.Close()
+
+ client, err := NewClient(ClientConfig{Host: server.URL, TokenName: "root@pam!token", TokenValue: "secret"})
+ if err != nil {
+ t.Fatalf("NewClient: %v", err)
+ }
+ evidence, err := client.GetJobHealthEvidence(context.Background(), []string{"main"}, JobHealthOptions{
+ MonitorSyncJobs: true,
+ MonitorVerifyJobs: true,
+ MonitorPruneJobs: true,
+ })
+ if err != nil {
+ t.Fatalf("GetJobHealthEvidence: %v", err)
+ }
+ byID := make(map[string]JobHealthEvidence)
+ for _, item := range evidence {
+ byID[item.ID] = item
+ }
+ if got := byID["prune-main-delly"]; got.LastRunState != "OK" || got.LastRunEndtime != 1793603400 || got.NextRun != 1793689800 || got.Confidence != JobEvidenceConfidenceConfigLastRun {
+ t.Fatalf("expected prune run status from /admin/prune, got %+v", got)
+ }
+ if got := byID["verify-main"]; got.LastRunState != "OK" || got.Confidence != JobEvidenceConfidenceConfigLastRun {
+ t.Fatalf("expected verify run status from /admin/verify, got %+v", got)
+ }
+ if got := byID["sync-offsite"]; got.Schedule != "hourly" || got.LastRunState != "" || got.Confidence != JobEvidenceConfidenceConfigOnly {
+ t.Fatalf("expected config-only sync evidence after /admin/sync was not found, got %+v", got)
+ }
+ if len(configReads) != 1 || configReads[0] != "/api2/json/config/sync" {
+ t.Fatalf("expected /config to be read only for the family without /admin, got %v", configReads)
+ }
+}
+
func TestPBSJobTaskHistoryQueries_UsesWorkerTypeFilters(t *testing.T) {
queries := buildTaskHistoryQueries([]string{"fast", "fast", "slow"}, JobHealthOptions{
MonitorBackups: true,