Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Keep unread PBS writer inventories from declaring failed backups
pulse-triage[bot]3c2aa3a9965f
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex c07a065cfe..189c5a53f6 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -17,6 +17,32 @@## Purpose+### Unread PBS writer inventories do not establish a terminal backup++The running backup/sync task reader requires an explicit `data` array from a+fully read response within the existing 8 MiB body bound. Missing/null arrays,+error-bearing envelopes, trailing JSON, unread/failed-close bodies and rows+without a nonblank worker type return no writer set, even after an earlier+family or page succeeded. Known non-writer types remain filtered; a missing+backup worker ID retains its conservative instance-wide match. Existing type+aliases, query filters, page bounds, permissions and polling budgets stay intact.++A failed writer read leaves this poll's write activity unobserved. An incomplete+snapshot remains non-successful and may still be running; the absence of task+evidence does not establish a terminal failure. An ordinary complete empty read+can still establish no writer, and subsequent unread data withdraws that fact.+Matched live tasks and completed manifests recover normally, without changing+snapshot identity or borrowing a completion time. This is not guest liveness,+a tested restore, an installed recovery or an established cause of #3248.++`TestClient_ListRunningDataTasksRequiresReadableInventory`, the later-page and+complete-response controls exercise the actual reader. Normal-suite+`TestPBSWriterInventoryDoesNotInventTerminalFailure` connects VM/CT HTTP reads+through polling, raw state, durable points and posture, including failed reads+after prior no-writer evidence and an unaffected completed sibling. Its opt-in+frames feed the actual existing Backups consumer at desktop/narrow widths for+interaction and native pixel proof. Fixtures perform only local GETs.+### Accepted PBS inventory survives its collection budget, not its ownerThe existing five-minute PBS backup budget bounds provider reads, not acceptancediff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.mdindex 4183926373..2011e16cf4 100644--- a/docs/release-control/v6/internal/subsystems/storage-recovery.md+++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md@@ -21,6 +21,32 @@## Purpose+### Unread PBS writer inventories do not establish a terminal backup++The running backup/sync task reader requires an explicit `data` array from a+fully read response within the existing 8 MiB body bound. Missing/null arrays,+error-bearing envelopes, trailing JSON, unread/failed-close bodies and rows+without a nonblank worker type return no writer set, even after an earlier+family or page succeeded. Known non-writer types remain filtered; a missing+backup worker ID retains its conservative instance-wide match. Existing type+aliases, query filters, page bounds, permissions and polling budgets stay intact.++A failed writer read leaves this poll's write activity unobserved. An incomplete+snapshot remains non-successful and may still be running; the absence of task+evidence does not establish a terminal failure. An ordinary complete empty read+can still establish no writer, and subsequent unread data withdraws that fact.+Matched live tasks and completed manifests recover normally, without changing+snapshot identity or borrowing a completion time. This is not guest liveness,+a tested restore, an installed recovery or an established cause of #3248.++`TestClient_ListRunningDataTasksRequiresReadableInventory`, the later-page and+complete-response controls exercise the actual reader. Normal-suite+`TestPBSWriterInventoryDoesNotInventTerminalFailure` connects VM/CT HTTP reads+through polling, raw state, durable points and posture, including failed reads+after prior no-writer evidence and an unaffected completed sibling. Its opt-in+frames feed the actual existing Backups consumer at desktop/narrow widths for+interaction and native pixel proof. Fixtures perform only local GETs.+### Coverage dates are inspectable without hoverOpen Coverage evidence retains each point’s exact reported date as wrappingdiff --git a/internal/monitoring/pbs_writer_inventory_test.go b/internal/monitoring/pbs_writer_inventory_test.gonew file mode 100644index 0000000000..2a1f6ca9c4--- /dev/null+++ b/internal/monitoring/pbs_writer_inventory_test.go@@ -0,0 +1,129 @@+package monitoring++import (+ "context"+ "encoding/json"+ "fmt"+ "net/http"+ "net/http/httptest"+ "os"+ "strings"+ "sync/atomic"+ "testing"++ "github.com/rcourtman/pulse-go-rewrite/internal/models"+ "github.com/rcourtman/pulse-go-rewrite/internal/recovery"+ "github.com/rcourtman/pulse-go-rewrite/pkg/pbs"+ "github.com/stretchr/testify/require"+)++// An unread writer inventory cannot turn a manifestless snapshot into a+// terminal failure. Exercise actual HTTP -> poll -> raw/durable/public state+// through ordinary recovery, including withdrawal of a prior no-writer fact.+// These fixtures perform no guest or backup operation.+func TestPBSWriterInventoryDoesNotInventTerminalFailure(t *testing.T) {+ for _, kind := range []string{"vm", "ct"} {+ for _, family := range []string{"backup", "syncjob"} {+ for _, invalid := range []string{`{}`, `{"data":null}`, `{"data":[{}]}`, `{"data":[null]}`, `{"data":[],"errors":{"private":"fixture-only"}}`} {+ t.Run(kind+"/"+family+"/"+invalid, func(t *testing.T) {+ f := newPBSPosturePollFixture(t, 2, kind)+ var phase atomic.Int32+ var writes atomic.Int32+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.Method != http.MethodGet {+ writes.Add(1)+ }+ switch {+ case strings.HasSuffix(r.URL.Path, "/groups"):+ _ = json.NewEncoder(w).Encode(map[string]any{"data": []map[string]any{+ {"backup-type": kind, "backup-id": "100", "last-backup": f.backupAt, "backup-count": 1},+ {"backup-type": kind, "backup-id": "101", "last-backup": f.backupAt, "backup-count": 1},+ }})+ case strings.HasSuffix(r.URL.Path, "/snapshots"):+ id := r.URL.Query().Get("backup-id")+ row := map[string]any{"backup-type": kind, "backup-id": id, "backup-time": f.backupAt,+ "size": 1024, "files": []string{"index.json.blob"}, "verification": map[string]any{"state": "ok"}}+ if id == "100" && phase.Load() != 5 {+ row["size"], row["files"] = 0, []string{"guest.conf.blob"}+ }+ _ = json.NewEncoder(w).Encode(map[string]any{"data": []map[string]any{row}})+ case strings.HasSuffix(r.URL.Path, "/tasks"):+ if (phase.Load() == 1 || phase.Load() == 3) && r.URL.Query().Get("typefilter") == family {+ _, _ = w.Write([]byte(invalid))+ } else if (phase.Load() == 0 || phase.Load() == 4) && r.URL.Query().Get("typefilter") == "backup" {+ _ = json.NewEncoder(w).Encode(map[string]any{"data": []map[string]any{{"worker_type": "backup", "worker_id": "archive:" + kind + "/100"}}})+ } else {+ _, _ = w.Write([]byte(`{"data":[]}`))+ }+ default:+ http.NotFound(w, r)+ }+ }))+ t.Cleanup(server.Close)+ client, err := pbs.NewClient(pbs.ClientConfig{Host: server.URL, TokenName: "fixture@pbs!reader", TokenValue: "fixture-only"})+ require.NoError(t, err)+ frames := map[string]any{}+ var identity string+ for i, label := range []string{"active", "unread", "empty", "unread-again", "active-again", "complete"} {+ phase.Store(int32(i))+ f.m.pollPBSBackups(context.Background(), "pbs-main", client, []models.PBSDatastore{{Name: "archive"}}, nil)+ f.wait(t)+ raw := f.m.state.GetSnapshot().PBSBackups+ require.Len(t, raw, 2)+ for _, b := range raw {+ if b.VMID == "101" {+ require.False(t, b.InProgress, "accepted sibling must not acquire writer uncertainty")+ continue+ }+ if identity == "" {+ identity = b.ID+ }+ wantedObservation := i == 0 || i == 2 || i == 4+ require.Equal(t, identity, b.ID, "ordinary polls must update the same snapshot")+ require.Equal(t, i != 5, b.InProgress)+ require.Equal(t, wantedObservation, b.WriteActivityObserved, label)+ require.Equal(t, i == 0 || i == 4, b.WriteActive, label)+ }+ points := f.points(t, "pbs-main")+ require.Len(t, points, 2)+ for _, p := range points {+ if fmt.Sprint(p.Details["vmid"]) == "101" {+ require.Equal(t, recovery.OutcomeSuccess, p.Outcome)+ continue+ }+ want := recovery.OutcomeRunning+ if i == 2 {+ want = recovery.OutcomeFailed+ } else if i == 5 {+ want = recovery.OutcomeSuccess+ }+ require.Equal(t, want, p.Outcome, label)+ if i != 5 {+ require.Nil(t, p.CompletedAt, "neither writer absence nor uncertainty supplies a completion time")+ require.Nil(t, p.Verified)+ }+ }+ postures := f.postures(t)+ for _, p := range postures {+ if p.SubjectResourceID == f.ids[0] && i != 5 {+ require.NotEqual(t, recovery.ProtectionStateProtected, p.State, "incomplete artifact cannot claim protection")+ }+ }+ frames[label] = map[string]any{+ "pbs": map[string]any{"data": map[string]any{"backups": raw}},+ "pve": map[string]any{"data": f.m.state.GetSnapshot().PVEBackups},+ "postures": map[string]any{"data": postures, "policy": recovery.DefaultProtectionPosturePolicy.Payload()},+ "points": points,+ }+ }+ require.Zero(t, writes.Load())+ if dir := os.Getenv("PULSE_PBS_WRITER_FRAMES"); dir != "" && invalid == `{}` && family == "syncjob" {+ data, err := json.Marshal(frames)+ require.NoError(t, err)+ require.NoError(t, os.WriteFile(dir+"/frames-"+kind+".json", data, 0600))+ }+ })+ }+ }+ }+}diff --git a/pkg/pbs/client.go b/pkg/pbs/client.goindex 182cee1b2d..24a0c61ed4 100644--- a/pkg/pbs/client.go+++ b/pkg/pbs/client.go@@ -1210,20 +1210,31 @@ func (c *Client) ListRunningDataTasks(ctx context.Context) ([]RunningDataTask, eif err != nil {return nil, err}- var result struct {- Data []map[string]interface{} `json:"data"`- }- decodeErr := json.NewDecoder(resp.Body).Decode(&result)+ body, readErr := readResponseBodyLimited(resp.Body)closeErr := resp.Body.Close()- if decodeErr != nil {- return nil, fmt.Errorf("failed to decode running PBS %s tasks: %w", requestedType, decodeErr)+ if readErr != nil {+ return nil, fmt.Errorf("failed to read running PBS %s tasks: %w", requestedType, readErr)}if closeErr != nil {return nil, fmt.Errorf("failed to close running PBS %s task response: %w", requestedType, closeErr)}+ var result struct {+ Data []map[string]interface{} `json:"data"`+ Errors json.RawMessage `json:"errors"`+ }+ if json.Unmarshal(body, &result) != nil || result.Data == nil ||+ (len(result.Errors) > 0 && !bytes.Equal(bytes.TrimSpace(result.Errors), []byte("null"))) {+ // Only an explicit, fully read inventory can establish no writer.+ // Keep provider bodies out of diagnostics and do not return a+ // partial writer set if a later family/page is unreadable.+ return nil, fmt.Errorf("invalid running PBS %s task inventory", requestedType)+ }for _, raw := range result.Data {workerType := strings.ToLower(stringJobField(raw, "worker-type", "worker_type", "type"))+ if workerType == "" {+ return nil, fmt.Errorf("running PBS %s task inventory contains no worker type", requestedType)+ }if workerType != requestedType {continue}diff --git a/pkg/pbs/client_http_test.go b/pkg/pbs/client_http_test.goindex dcf4665d74..f6023a9c03 100644--- a/pkg/pbs/client_http_test.go+++ b/pkg/pbs/client_http_test.go@@ -145,6 +145,111 @@ func TestClient_ListRunningDataTasksRejectsTruncatedWriterSet(t *testing.T) {}}+func TestClient_ListRunningDataTasksRequiresReadableInventory(t *testing.T) {+ for _, family := range []string{"backup", "syncjob"} {+ for _, body := range []string{+ `null`, `{}`, `{"data":null}`, `{"data":{}}`, `{"data":false}`, `{"data":`,+ `{"data":[null]}`, `{"data":[{}]}`, `{"data":[{"worker_type":null}]}`,+ `{"data":[{"worker_type":123}]}`, `{"data":[{"worker_type":" "}]}`,+ `{"data":[],"errors":{"private":"provider detail"}}`, `{"data":[]} {"data":[]}`,+ } {+ t.Run(family+"/"+body, func(t *testing.T) {+ reads := 0+ client, server := newTestClient(t, func(w http.ResponseWriter, r *http.Request) {+ reads+++ if r.Method != http.MethodGet || r.URL.Query().Get("running") != "true" {+ t.Errorf("unexpected writer read: %s %s", r.Method, r.URL)+ }+ if r.URL.Query().Get("typefilter") == family {+ _, _ = w.Write([]byte(body))+ } else {+ _, _ = w.Write([]byte(`{"data":[{"worker_type":"backup","worker_id":"archive:vm/100"}]}`))+ }+ })+ defer server.Close()+ tasks, err := client.ListRunningDataTasks(context.Background())+ if err == nil || tasks != nil {+ t.Fatalf("unread inventory accepted or returned partial writers: tasks=%v err=%v", tasks, err)+ }+ if strings.Contains(err.Error(), "provider detail") {+ t.Fatal("provider body leaked into format diagnostic")+ }+ wantReads := 1+ if family == "syncjob" {+ wantReads = 2+ }+ if reads != wantReads {+ t.Fatalf("reads=%d want=%d; invalid inventories must not retry", reads, wantReads)+ }+ })+ }+ }+ for _, body := range []string{`{"data":[]}`, `{"data":[],"errors":null,"total":0}`} {+ t.Run("explicit-empty/"+body, func(t *testing.T) {+ client, server := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte(body)) })+ defer server.Close()+ tasks, err := client.ListRunningDataTasks(context.Background())+ if err != nil || len(tasks) != 0 {+ t.Fatalf("explicit no-writer inventory rejected: tasks=%v err=%v", tasks, err)+ }+ })+ }+}++func TestClient_ListRunningDataTasksRejectsUnreadLaterPage(t *testing.T) {+ client, server := newTestClient(t, func(w http.ResponseWriter, r *http.Request) {+ if r.URL.Query().Get("start") != "" {+ _, _ = w.Write([]byte(`{"data":null}`))+ return+ }+ rows := make([]map[string]string, pbsRunningTaskPageLimit)+ for i := range rows {+ rows[i] = map[string]string{"worker_type": "backup", "worker_id": "archive:vm/100"}+ }+ _ = json.NewEncoder(w).Encode(map[string]any{"data": rows})+ })+ defer server.Close()+ tasks, err := client.ListRunningDataTasks(context.Background())+ if err == nil || tasks != nil {+ t.Fatalf("unread later page accepted as complete writers: tasks=%v err=%v", tasks, err)+ }+}++type runningTasksBody struct {+ io.Reader+ closeErr error+ closed bool+}++func (b *runningTasksBody) Close() error { b.closed = true; return b.closeErr }++func TestClient_ListRunningDataTasksRequiresCompleteResponse(t *testing.T) {+ for _, failure := range []string{"read", "close", "size"} {+ t.Run(failure, func(t *testing.T) {+ body := &runningTasksBody{Reader: strings.NewReader(`{"data":[]}`)}+ switch failure {+ case "read":+ body.Reader = io.MultiReader(body.Reader, &failReader{})+ case "close":+ body.closeErr = errors.New("fixture close failure")+ case "size":+ body.Reader = io.MultiReader(body.Reader, strings.NewReader(strings.Repeat(" ", int(maxResponseBodyBytes))))+ }+ client, err := NewClient(ClientConfig{Host: "https://pbs.invalid", TokenName: "fixture@pbs!reader", TokenValue: "fixture-only"})+ if err != nil {+ t.Fatal(err)+ }+ client.httpClient.Transport = roundTripFunc(func(*http.Request) (*http.Response, error) {+ return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: body}, nil+ })+ tasks, err := client.ListRunningDataTasks(context.Background())+ if err == nil || tasks != nil || !body.closed {+ t.Fatalf("incomplete response accepted/unclosed: tasks=%v err=%v closed=%t", tasks, err, body.closed)+ }+ })+ }+}+func TestClient_GetJobHealthEvidence_MergesConfigAndTaskFacts(t *testing.T) {server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {switch r.URL.Path {