Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Keep unread PBS writer inventories from declaring failed backups

pulse-triage[bot]3c2aa3a9965f
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index c07a065cfe..189c5a53f6 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -17,6 +17,32 @@
## Purpose
+### Unread PBS writer inventories do not establish a terminal backup
+
+The running backup/sync task reader requires an explicit `data` array from a
+fully read response within the existing 8 MiB body bound. Missing/null arrays,
+error-bearing envelopes, trailing JSON, unread/failed-close bodies and rows
+without a nonblank worker type return no writer set, even after an earlier
+family or page succeeded. Known non-writer types remain filtered; a missing
+backup worker ID retains its conservative instance-wide match. Existing type
+aliases, query filters, page bounds, permissions and polling budgets stay intact.
+
+A failed writer read leaves this poll's write activity unobserved. An incomplete
+snapshot remains non-successful and may still be running; the absence of task
+evidence does not establish a terminal failure. An ordinary complete empty read
+can still establish no writer, and subsequent unread data withdraws that fact.
+Matched live tasks and completed manifests recover normally, without changing
+snapshot identity or borrowing a completion time. This is not guest liveness,
+a tested restore, an installed recovery or an established cause of #3248.
+
+`TestClient_ListRunningDataTasksRequiresReadableInventory`, the later-page and
+complete-response controls exercise the actual reader. Normal-suite
+`TestPBSWriterInventoryDoesNotInventTerminalFailure` connects VM/CT HTTP reads
+through polling, raw state, durable points and posture, including failed reads
+after prior no-writer evidence and an unaffected completed sibling. Its opt-in
+frames feed the actual existing Backups consumer at desktop/narrow widths for
+interaction and native pixel proof. Fixtures perform only local GETs.
+
### Accepted PBS inventory survives its collection budget, not its owner
The existing five-minute PBS backup budget bounds provider reads, not acceptance
diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md
index 4183926373..2011e16cf4 100644
--- a/docs/release-control/v6/internal/subsystems/storage-recovery.md
+++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md
@@ -21,6 +21,32 @@
## Purpose
+### Unread PBS writer inventories do not establish a terminal backup
+
+The running backup/sync task reader requires an explicit `data` array from a
+fully read response within the existing 8 MiB body bound. Missing/null arrays,
+error-bearing envelopes, trailing JSON, unread/failed-close bodies and rows
+without a nonblank worker type return no writer set, even after an earlier
+family or page succeeded. Known non-writer types remain filtered; a missing
+backup worker ID retains its conservative instance-wide match. Existing type
+aliases, query filters, page bounds, permissions and polling budgets stay intact.
+
+A failed writer read leaves this poll's write activity unobserved. An incomplete
+snapshot remains non-successful and may still be running; the absence of task
+evidence does not establish a terminal failure. An ordinary complete empty read
+can still establish no writer, and subsequent unread data withdraws that fact.
+Matched live tasks and completed manifests recover normally, without changing
+snapshot identity or borrowing a completion time. This is not guest liveness,
+a tested restore, an installed recovery or an established cause of #3248.
+
+`TestClient_ListRunningDataTasksRequiresReadableInventory`, the later-page and
+complete-response controls exercise the actual reader. Normal-suite
+`TestPBSWriterInventoryDoesNotInventTerminalFailure` connects VM/CT HTTP reads
+through polling, raw state, durable points and posture, including failed reads
+after prior no-writer evidence and an unaffected completed sibling. Its opt-in
+frames feed the actual existing Backups consumer at desktop/narrow widths for
+interaction and native pixel proof. Fixtures perform only local GETs.
+
### Coverage dates are inspectable without hover
Open Coverage evidence retains each point’s exact reported date as wrapping
diff --git a/internal/monitoring/pbs_writer_inventory_test.go b/internal/monitoring/pbs_writer_inventory_test.go
new file mode 100644
index 0000000000..2a1f6ca9c4
--- /dev/null
+++ b/internal/monitoring/pbs_writer_inventory_test.go
@@ -0,0 +1,129 @@
+package monitoring
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "strings"
+ "sync/atomic"
+ "testing"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/models"
+ "github.com/rcourtman/pulse-go-rewrite/internal/recovery"
+ "github.com/rcourtman/pulse-go-rewrite/pkg/pbs"
+ "github.com/stretchr/testify/require"
+)
+
+// An unread writer inventory cannot turn a manifestless snapshot into a
+// terminal failure. Exercise actual HTTP -> poll -> raw/durable/public state
+// through ordinary recovery, including withdrawal of a prior no-writer fact.
+// These fixtures perform no guest or backup operation.
+func TestPBSWriterInventoryDoesNotInventTerminalFailure(t *testing.T) {
+ for _, kind := range []string{"vm", "ct"} {
+ for _, family := range []string{"backup", "syncjob"} {
+ for _, invalid := range []string{`{}`, `{"data":null}`, `{"data":[{}]}`, `{"data":[null]}`, `{"data":[],"errors":{"private":"fixture-only"}}`} {
+ t.Run(kind+"/"+family+"/"+invalid, func(t *testing.T) {
+ f := newPBSPosturePollFixture(t, 2, kind)
+ var phase atomic.Int32
+ var writes atomic.Int32
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.Method != http.MethodGet {
+ writes.Add(1)
+ }
+ switch {
+ case strings.HasSuffix(r.URL.Path, "/groups"):
+ _ = json.NewEncoder(w).Encode(map[string]any{"data": []map[string]any{
+ {"backup-type": kind, "backup-id": "100", "last-backup": f.backupAt, "backup-count": 1},
+ {"backup-type": kind, "backup-id": "101", "last-backup": f.backupAt, "backup-count": 1},
+ }})
+ case strings.HasSuffix(r.URL.Path, "/snapshots"):
+ id := r.URL.Query().Get("backup-id")
+ row := map[string]any{"backup-type": kind, "backup-id": id, "backup-time": f.backupAt,
+ "size": 1024, "files": []string{"index.json.blob"}, "verification": map[string]any{"state": "ok"}}
+ if id == "100" && phase.Load() != 5 {
+ row["size"], row["files"] = 0, []string{"guest.conf.blob"}
+ }
+ _ = json.NewEncoder(w).Encode(map[string]any{"data": []map[string]any{row}})
+ case strings.HasSuffix(r.URL.Path, "/tasks"):
+ if (phase.Load() == 1 || phase.Load() == 3) && r.URL.Query().Get("typefilter") == family {
+ _, _ = w.Write([]byte(invalid))
+ } else if (phase.Load() == 0 || phase.Load() == 4) && r.URL.Query().Get("typefilter") == "backup" {
+ _ = json.NewEncoder(w).Encode(map[string]any{"data": []map[string]any{{"worker_type": "backup", "worker_id": "archive:" + kind + "/100"}}})
+ } else {
+ _, _ = w.Write([]byte(`{"data":[]}`))
+ }
+ default:
+ http.NotFound(w, r)
+ }
+ }))
+ t.Cleanup(server.Close)
+ client, err := pbs.NewClient(pbs.ClientConfig{Host: server.URL, TokenName: "fixture@pbs!reader", TokenValue: "fixture-only"})
+ require.NoError(t, err)
+ frames := map[string]any{}
+ var identity string
+ for i, label := range []string{"active", "unread", "empty", "unread-again", "active-again", "complete"} {
+ phase.Store(int32(i))
+ f.m.pollPBSBackups(context.Background(), "pbs-main", client, []models.PBSDatastore{{Name: "archive"}}, nil)
+ f.wait(t)
+ raw := f.m.state.GetSnapshot().PBSBackups
+ require.Len(t, raw, 2)
+ for _, b := range raw {
+ if b.VMID == "101" {
+ require.False(t, b.InProgress, "accepted sibling must not acquire writer uncertainty")
+ continue
+ }
+ if identity == "" {
+ identity = b.ID
+ }
+ wantedObservation := i == 0 || i == 2 || i == 4
+ require.Equal(t, identity, b.ID, "ordinary polls must update the same snapshot")
+ require.Equal(t, i != 5, b.InProgress)
+ require.Equal(t, wantedObservation, b.WriteActivityObserved, label)
+ require.Equal(t, i == 0 || i == 4, b.WriteActive, label)
+ }
+ points := f.points(t, "pbs-main")
+ require.Len(t, points, 2)
+ for _, p := range points {
+ if fmt.Sprint(p.Details["vmid"]) == "101" {
+ require.Equal(t, recovery.OutcomeSuccess, p.Outcome)
+ continue
+ }
+ want := recovery.OutcomeRunning
+ if i == 2 {
+ want = recovery.OutcomeFailed
+ } else if i == 5 {
+ want = recovery.OutcomeSuccess
+ }
+ require.Equal(t, want, p.Outcome, label)
+ if i != 5 {
+ require.Nil(t, p.CompletedAt, "neither writer absence nor uncertainty supplies a completion time")
+ require.Nil(t, p.Verified)
+ }
+ }
+ postures := f.postures(t)
+ for _, p := range postures {
+ if p.SubjectResourceID == f.ids[0] && i != 5 {
+ require.NotEqual(t, recovery.ProtectionStateProtected, p.State, "incomplete artifact cannot claim protection")
+ }
+ }
+ frames[label] = map[string]any{
+ "pbs": map[string]any{"data": map[string]any{"backups": raw}},
+ "pve": map[string]any{"data": f.m.state.GetSnapshot().PVEBackups},
+ "postures": map[string]any{"data": postures, "policy": recovery.DefaultProtectionPosturePolicy.Payload()},
+ "points": points,
+ }
+ }
+ require.Zero(t, writes.Load())
+ if dir := os.Getenv("PULSE_PBS_WRITER_FRAMES"); dir != "" && invalid == `{}` && family == "syncjob" {
+ data, err := json.Marshal(frames)
+ require.NoError(t, err)
+ require.NoError(t, os.WriteFile(dir+"/frames-"+kind+".json", data, 0600))
+ }
+ })
+ }
+ }
+ }
+}
diff --git a/pkg/pbs/client.go b/pkg/pbs/client.go
index 182cee1b2d..24a0c61ed4 100644
--- a/pkg/pbs/client.go
+++ b/pkg/pbs/client.go
@@ -1210,20 +1210,31 @@ func (c *Client) ListRunningDataTasks(ctx context.Context) ([]RunningDataTask, e
if err != nil {
return nil, err
}
- var result struct {
- Data []map[string]interface{} `json:"data"`
- }
- decodeErr := json.NewDecoder(resp.Body).Decode(&result)
+ body, readErr := readResponseBodyLimited(resp.Body)
closeErr := resp.Body.Close()
- if decodeErr != nil {
- return nil, fmt.Errorf("failed to decode running PBS %s tasks: %w", requestedType, decodeErr)
+ if readErr != nil {
+ return nil, fmt.Errorf("failed to read running PBS %s tasks: %w", requestedType, readErr)
}
if closeErr != nil {
return nil, fmt.Errorf("failed to close running PBS %s task response: %w", requestedType, closeErr)
}
+ var result struct {
+ Data []map[string]interface{} `json:"data"`
+ Errors json.RawMessage `json:"errors"`
+ }
+ if json.Unmarshal(body, &result) != nil || result.Data == nil ||
+ (len(result.Errors) > 0 && !bytes.Equal(bytes.TrimSpace(result.Errors), []byte("null"))) {
+ // Only an explicit, fully read inventory can establish no writer.
+ // Keep provider bodies out of diagnostics and do not return a
+ // partial writer set if a later family/page is unreadable.
+ return nil, fmt.Errorf("invalid running PBS %s task inventory", requestedType)
+ }
for _, raw := range result.Data {
workerType := strings.ToLower(stringJobField(raw, "worker-type", "worker_type", "type"))
+ if workerType == "" {
+ return nil, fmt.Errorf("running PBS %s task inventory contains no worker type", requestedType)
+ }
if workerType != requestedType {
continue
}
diff --git a/pkg/pbs/client_http_test.go b/pkg/pbs/client_http_test.go
index dcf4665d74..f6023a9c03 100644
--- a/pkg/pbs/client_http_test.go
+++ b/pkg/pbs/client_http_test.go
@@ -145,6 +145,111 @@ func TestClient_ListRunningDataTasksRejectsTruncatedWriterSet(t *testing.T) {
}
}
+func TestClient_ListRunningDataTasksRequiresReadableInventory(t *testing.T) {
+ for _, family := range []string{"backup", "syncjob"} {
+ for _, body := range []string{
+ `null`, `{}`, `{"data":null}`, `{"data":{}}`, `{"data":false}`, `{"data":`,
+ `{"data":[null]}`, `{"data":[{}]}`, `{"data":[{"worker_type":null}]}`,
+ `{"data":[{"worker_type":123}]}`, `{"data":[{"worker_type":" "}]}`,
+ `{"data":[],"errors":{"private":"provider detail"}}`, `{"data":[]} {"data":[]}`,
+ } {
+ t.Run(family+"/"+body, func(t *testing.T) {
+ reads := 0
+ client, server := newTestClient(t, func(w http.ResponseWriter, r *http.Request) {
+ reads++
+ if r.Method != http.MethodGet || r.URL.Query().Get("running") != "true" {
+ t.Errorf("unexpected writer read: %s %s", r.Method, r.URL)
+ }
+ if r.URL.Query().Get("typefilter") == family {
+ _, _ = w.Write([]byte(body))
+ } else {
+ _, _ = w.Write([]byte(`{"data":[{"worker_type":"backup","worker_id":"archive:vm/100"}]}`))
+ }
+ })
+ defer server.Close()
+ tasks, err := client.ListRunningDataTasks(context.Background())
+ if err == nil || tasks != nil {
+ t.Fatalf("unread inventory accepted or returned partial writers: tasks=%v err=%v", tasks, err)
+ }
+ if strings.Contains(err.Error(), "provider detail") {
+ t.Fatal("provider body leaked into format diagnostic")
+ }
+ wantReads := 1
+ if family == "syncjob" {
+ wantReads = 2
+ }
+ if reads != wantReads {
+ t.Fatalf("reads=%d want=%d; invalid inventories must not retry", reads, wantReads)
+ }
+ })
+ }
+ }
+ for _, body := range []string{`{"data":[]}`, `{"data":[],"errors":null,"total":0}`} {
+ t.Run("explicit-empty/"+body, func(t *testing.T) {
+ client, server := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte(body)) })
+ defer server.Close()
+ tasks, err := client.ListRunningDataTasks(context.Background())
+ if err != nil || len(tasks) != 0 {
+ t.Fatalf("explicit no-writer inventory rejected: tasks=%v err=%v", tasks, err)
+ }
+ })
+ }
+}
+
+func TestClient_ListRunningDataTasksRejectsUnreadLaterPage(t *testing.T) {
+ client, server := newTestClient(t, func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Query().Get("start") != "" {
+ _, _ = w.Write([]byte(`{"data":null}`))
+ return
+ }
+ rows := make([]map[string]string, pbsRunningTaskPageLimit)
+ for i := range rows {
+ rows[i] = map[string]string{"worker_type": "backup", "worker_id": "archive:vm/100"}
+ }
+ _ = json.NewEncoder(w).Encode(map[string]any{"data": rows})
+ })
+ defer server.Close()
+ tasks, err := client.ListRunningDataTasks(context.Background())
+ if err == nil || tasks != nil {
+ t.Fatalf("unread later page accepted as complete writers: tasks=%v err=%v", tasks, err)
+ }
+}
+
+type runningTasksBody struct {
+ io.Reader
+ closeErr error
+ closed bool
+}
+
+func (b *runningTasksBody) Close() error { b.closed = true; return b.closeErr }
+
+func TestClient_ListRunningDataTasksRequiresCompleteResponse(t *testing.T) {
+ for _, failure := range []string{"read", "close", "size"} {
+ t.Run(failure, func(t *testing.T) {
+ body := &runningTasksBody{Reader: strings.NewReader(`{"data":[]}`)}
+ switch failure {
+ case "read":
+ body.Reader = io.MultiReader(body.Reader, &failReader{})
+ case "close":
+ body.closeErr = errors.New("fixture close failure")
+ case "size":
+ body.Reader = io.MultiReader(body.Reader, strings.NewReader(strings.Repeat(" ", int(maxResponseBodyBytes))))
+ }
+ client, err := NewClient(ClientConfig{Host: "https://pbs.invalid", TokenName: "fixture@pbs!reader", TokenValue: "fixture-only"})
+ if err != nil {
+ t.Fatal(err)
+ }
+ client.httpClient.Transport = roundTripFunc(func(*http.Request) (*http.Response, error) {
+ return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: body}, nil
+ })
+ tasks, err := client.ListRunningDataTasks(context.Background())
+ if err == nil || tasks != nil || !body.closed {
+ t.Fatalf("incomplete response accepted/unclosed: tasks=%v err=%v closed=%t", tasks, err, body.closed)
+ }
+ })
+ }
+}
+
func TestClient_GetJobHealthEvidence_MergesConfigAndTaskFacts(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {