Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Require whole PBS inventory responses before reconciling history

pulse-triage[bot]0d2aa2a731aa
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index 189c5a53f6..bd14594ef4 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -17,6 +17,26 @@
## Purpose
+### PBS backup listings require the whole response
+
+Namespace, group and snapshot reads accept one unambiguous JSON envelope only
+from a fully read and successfully closed HTTP 200 body within the existing
+8 MiB bound. Trailing values/garbage, invalid UTF-8, duplicate envelope fields,
+case aliases of `data`/`errors`, and non-null error envelopes are unread, not
+accepted empty inventory or fresh protection evidence. Unique future metadata
+and explicit empty arrays remain valid; the older-PBS namespace 404 fallback is
+unchanged. Diagnostics contain no provider payload. No retry, auth, poll budget,
+identity, guest operation or retention policy changes.
+
+Normal client suites cover body/close/bound failures and envelope controls.
+`TestPBSWholeInventoryResponseRetainsEvidence` exercises VM/CT cold/warm HTTP
+collection through live state, cache times, durable points and posture: unread
+scopes retain old evidence without renewing it, readable siblings advance, and
+ordinary complete/empty reads recover and reconcile absence.
+`TestPBSWholeInventoryConsumerLifecycle` feeds those actual poll results to
+source-bound desktop/narrow Backups inspection. This is truthful collection,
+not original guest recovery, an established #3248 cause or release availability.
+
### Unread PBS writer inventories do not establish a terminal backup
The running backup/sync task reader requires an explicit `data` array from a
diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md
index 2011e16cf4..877da1baef 100644
--- a/docs/release-control/v6/internal/subsystems/storage-recovery.md
+++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md
@@ -21,6 +21,26 @@
## Purpose
+### PBS backup listings require the whole response
+
+Namespace, group and snapshot reads accept one unambiguous JSON envelope only
+from a fully read and successfully closed HTTP 200 body within the existing
+8 MiB bound. Trailing values/garbage, invalid UTF-8, duplicate envelope fields,
+case aliases of `data`/`errors`, and non-null error envelopes are unread, not
+accepted empty inventory or fresh protection evidence. Unique future metadata
+and explicit empty arrays remain valid; the older-PBS namespace 404 fallback is
+unchanged. Diagnostics contain no provider payload. No retry, auth, poll budget,
+identity, guest operation or retention policy changes.
+
+Normal client suites cover body/close/bound failures and envelope controls.
+`TestPBSWholeInventoryResponseRetainsEvidence` exercises VM/CT cold/warm HTTP
+collection through live state, cache times, durable points and posture: unread
+scopes retain old evidence without renewing it, readable siblings advance, and
+ordinary complete/empty reads recover and reconcile absence.
+`TestPBSWholeInventoryConsumerLifecycle` feeds those actual poll results to
+source-bound desktop/narrow Backups inspection. This is truthful collection,
+not original guest recovery, an established #3248 cause or release availability.
+
### Unread PBS writer inventories do not establish a terminal backup
The running backup/sync task reader requires an explicit `data` array from a
diff --git a/internal/monitoring/pbs_inventory_response_test.go b/internal/monitoring/pbs_inventory_response_test.go
new file mode 100644
index 0000000000..1d29d37c3a
--- /dev/null
+++ b/internal/monitoring/pbs_inventory_response_test.go
@@ -0,0 +1,169 @@
+package monitoring
+
+import (
+ "crypto/sha256"
+ "encoding/json"
+ "fmt"
+ "os"
+ "reflect"
+ "testing"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust"
+ "github.com/rcourtman/pulse-go-rewrite/internal/recovery"
+)
+
+func TestPBSWholeInventoryResponseRetainsEvidence(t *testing.T) {
+ for _, kind := range []string{"vm", "ct"} {
+ for _, stage := range []string{"namespace", "groups", "snapshots"} {
+ for name, body := range map[string]string{
+ "trailing value": `{"data":[]} {"data":[]}`,
+ "error envelope": `{"data":[],"errors":{"private":"fixture-only"}}`,
+ "overwritten data": `{"data":null,"data":[]}`,
+ "invalid utf8": "{\"data\":[],\"metadata\":\"\xff\"}",
+ } {
+ for _, warm := range []bool{false, true} {
+ t.Run(fmt.Sprintf("%s/%s/%s/warm=%t", kind, stage, name, warm), func(t *testing.T) {
+ f := newPBSNamespaceContinuityFixture(t, kind)
+ if warm {
+ f.collect(t, true, nil)
+ }
+ before, points := f.live(), f.points(t, "pbs-main")
+ key := pbsBackupGroupKey{datastore: "archive", namespace: "tenant", backupType: kind, backupID: "101"}
+ cacheAt := f.m.pbsBackupCacheTimeFor("pbs-main", key)
+ f.rootAt.Add(15 * 60)
+ f.tenantAt.Add(15 * 60)
+ for range 2 {
+ f.collectInventoryResponse(t, stage, body)
+ rows := f.live()
+ want := 1
+ if warm {
+ want = 2
+ if !reflect.DeepEqual(rows["101"], before["101"]) {
+ t.Error("unread response removed or changed accepted live file")
+ }
+ }
+ if len(rows) != want || rows["100"].BackupTime.Unix() != f.rootAt.Load() {
+ t.Error("unread response invented/lost files or held back readable sibling")
+ }
+ if !f.m.pbsBackupCacheTimeFor("pbs-main", key).Equal(cacheAt) {
+ t.Error("unread response renewed snapshot cache time")
+ }
+ for _, prior := range points {
+ if prior.Details["namespace"] != "tenant" {
+ continue
+ }
+ found := false
+ for _, now := range f.points(t, "pbs-main") {
+ found = found || (prior.ID == now.ID && reflect.DeepEqual(prior, now))
+ }
+ if !found {
+ t.Error("unread response removed or renewed durable evidence")
+ }
+ }
+ for _, p := range f.postures(t) {
+ if !warm && p.SubjectResourceID == f.ids[1] {
+ if p.State != recovery.ProtectionStateUnknown || len(p.ProviderStates) != 0 {
+ t.Error("cold unread scope acquired evidence")
+ }
+ continue
+ }
+ if p.State == recovery.ProtectionStateProtected || len(p.ProviderStates) != 1 || p.ProviderStates[0].HistoryCompleteness != recovery.ProtectionHistoryPartial || p.ProviderStates[0].Permissions != operationaltrust.EvidencePermissionsUnknown {
+ t.Errorf("unread response claimed protection or denied access: %+v", p)
+ }
+ }
+ }
+ if f.failedReads.Load() != 2 {
+ t.Error("invalid response caused extra reads")
+ }
+ f.inventoryResponse.Store(nil)
+ f.collect(t, true, nil)
+ if f.live()["101"].BackupTime.Unix() != f.tenantAt.Load() {
+ t.Error("complete ordinary read failed to recover")
+ }
+ for _, p := range f.postures(t) {
+ if p.State != recovery.ProtectionStateProtected {
+ t.Error("ordinary recovery failed to establish complete protection")
+ }
+ }
+ f.rootEmpty.Store(true)
+ f.tenantEmpty.Store(true)
+ f.collect(t, true, nil)
+ if len(f.live()) != 0 || len(f.points(t, "pbs-main")) != 0 {
+ t.Error("complete empty read failed to reconcile confirmed absence")
+ }
+ })
+ }
+ }
+ }
+ }
+}
+
+// Exercise ordinary polling, raw publication, durable ingestion and posture.
+// Optional export is evidence for the real Backups consumer, not a second path.
+func TestPBSWholeInventoryConsumerLifecycle(t *testing.T) {
+ for _, kind := range []string{"vm", "ct"} {
+ t.Run(kind, func(t *testing.T) {
+ f := newPBSNamespaceContinuityFixture(t, kind)
+ frames := map[string]any{}
+ for _, phase := range []string{"observed", "unread-namespaces", "unread-groups", "unread-snapshots", "recovered", "confirmed-empty"} {
+ switch phase {
+ case "unread-namespaces":
+ f.rootAt.Add(15 * 60)
+ f.collectInventoryResponse(t, "namespace", `{"data":[]} {"data":[]}`)
+ case "unread-groups":
+ f.collectInventoryResponse(t, "groups", `{"data":[],"errors":{"private":"fixture-only"}}`)
+ case "unread-snapshots":
+ f.tenantAt.Add(15 * 60)
+ f.collectInventoryResponse(t, "snapshots", `{"data":null,"data":[]}`)
+ default:
+ f.inventoryResponse.Store(nil)
+ if phase == "confirmed-empty" {
+ f.rootEmpty.Store(true)
+ f.tenantEmpty.Store(true)
+ }
+ f.collect(t, true, nil)
+ }
+ for _, p := range f.postures(t) {
+ if p.SubjectResourceID == f.ids[1] || p.SubjectResourceID == f.ids[0] {
+ if (phase == "observed" || phase == "recovered") != (p.State == recovery.ProtectionStateProtected) {
+ t.Fatalf("phase %s posture=%+v", phase, p)
+ }
+ }
+ }
+ want := 2
+ wantPoints := 2
+ if phase == "confirmed-empty" {
+ want = 0
+ wantPoints = 0
+ } else if phase == "unread-namespaces" || phase == "unread-groups" || phase == "unread-snapshots" {
+ wantPoints = 3
+ }
+ if len(f.live()) != want || len(f.points(t, "pbs-main")) != wantPoints {
+ t.Fatalf("phase %s lost or invented inventory", phase)
+ }
+ frames[phase] = map[string]any{
+ "pbs": map[string]any{"data": map[string]any{"backups": f.m.state.GetSnapshot().PBSBackups}},
+ "postures": map[string]any{"data": f.postures(t), "policy": recovery.DefaultProtectionPosturePolicy.Payload()},
+ "points": f.points(t, "pbs-main"),
+ }
+ }
+ if dest := os.Getenv("PULSE_PBS_RESPONSE_FRAMES"); dest != "" {
+ hashes := map[string]string{}
+ for _, file := range []string{"monitor_backups.go", "backup_guard.go", "../../pkg/pbs/client.go", "../../pkg/pbs/backup_inventory_response.go"} {
+ content, err := os.ReadFile(file)
+ if err != nil {
+ t.Fatal(err)
+ }
+ hashes[file] = fmt.Sprintf("%x", sha256.Sum256(content))
+ }
+ data, err := json.Marshal(map[string]any{"runtime_sha256": hashes, "resource_ids": f.ids, "frames": frames})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(dest+"/frames-"+kind+".json", data, 0600); err != nil {
+ t.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/pbs/backup_inventory_response.go b/pkg/pbs/backup_inventory_response.go
new file mode 100644
index 0000000000..6017939bcc
--- /dev/null
+++ b/pkg/pbs/backup_inventory_response.go
@@ -0,0 +1,72 @@
+package pbs
+
+import (
+ "bytes"
+ "encoding/json"
+ "errors"
+ "io"
+ "net/http"
+ "strings"
+ "unicode/utf8"
+)
+
+var errInvalidBackupInventoryResponse = errors.New("invalid PBS backup inventory response")
+
+// A list can reconcile absence or renew backup protection. Accept only one
+// unambiguous envelope from a fully read, bounded response, never a JSON prefix.
+// Keep provider payloads out of format diagnostics (including reported errors).
+func readBackupInventory[T any](resp *http.Response) ([]T, error) {
+ body, readErr := readResponseBodyLimited(resp.Body)
+ closeErr := resp.Body.Close()
+ if readErr != nil || closeErr != nil {
+ return nil, errors.Join(readErr, closeErr)
+ }
+ if resp.StatusCode != http.StatusOK || !utf8.Valid(body) {
+ return nil, errInvalidBackupInventoryResponse
+ }
+ decoder := json.NewDecoder(bytes.NewReader(body))
+ start, err := decoder.Token()
+ if err != nil || start != json.Delim('{') {
+ return nil, errInvalidBackupInventoryResponse
+ }
+ fields := make(map[string]json.RawMessage)
+ for decoder.More() {
+ token, err := decoder.Token()
+ if err != nil {
+ return nil, errInvalidBackupInventoryResponse
+ }
+ name, ok := token.(string)
+ if !ok {
+ return nil, errInvalidBackupInventoryResponse
+ }
+ if _, duplicate := fields[name]; duplicate ||
+ (name != "data" && strings.EqualFold(name, "data")) ||
+ (name != "errors" && strings.EqualFold(name, "errors")) {
+ return nil, errInvalidBackupInventoryResponse
+ }
+ var value json.RawMessage
+ if decoder.Decode(&value) != nil {
+ return nil, errInvalidBackupInventoryResponse
+ }
+ fields[name] = value
+ }
+ end, err := decoder.Token()
+ if err != nil || end != json.Delim('}') {
+ return nil, errInvalidBackupInventoryResponse
+ }
+ if _, err := decoder.Token(); err != io.EOF {
+ return nil, errInvalidBackupInventoryResponse
+ }
+ if failure, present := fields["errors"]; present && !bytes.Equal(bytes.TrimSpace(failure), []byte("null")) {
+ return nil, errInvalidBackupInventoryResponse
+ }
+ data := bytes.TrimSpace(fields["data"])
+ if len(data) == 0 || data[0] != '[' {
+ return nil, errInvalidBackupInventoryResponse
+ }
+ var rows []T
+ if json.Unmarshal(data, &rows) != nil {
+ return nil, errInvalidBackupInventoryResponse
+ }
+ return rows, nil
+}
diff --git a/pkg/pbs/client.go b/pkg/pbs/client.go
index 24a0c61ed4..894c742144 100644
--- a/pkg/pbs/client.go
+++ b/pkg/pbs/client.go
@@ -1966,22 +1966,7 @@ func (c *Client) ListNamespaces(ctx context.Context, datastore string, parentNam
}
return nil, err
}
- defer resp.Body.Close()
-
- var result struct {
- Data []Namespace `json:"data"`
- }
-
- if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
- return nil, err
- }
- // Match datastore discovery: missing/null data is unread, not an empty
- // scope that can authorize removal of retained backup history.
- if result.Data == nil {
- return nil, fmt.Errorf("namespace list response contains no data array")
- }
-
- return result.Data, nil
+ return readBackupInventory[Namespace](resp)
}
// ListBackupGroups lists all backup groups in a datastore/namespace
@@ -2005,31 +1990,16 @@ func (c *Client) ListBackupGroups(ctx context.Context, datastore string, namespa
if err != nil {
return nil, err
}
- defer resp.Body.Close()
-
- if resp.StatusCode != http.StatusOK {
- body, err := readResponseBodyLimited(resp.Body)
- if err != nil {
- return nil, err
- }
- return nil, fmt.Errorf("unexpected status %d: %s", resp.StatusCode, string(body))
- }
-
- var result struct {
- Data []BackupGroup `json:"data"`
- }
- if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
- return nil, fmt.Errorf("failed to decode backup groups: %w", err)
- }
- if result.Data == nil {
- return nil, fmt.Errorf("backup group list response contains no data array")
+ groups, err := readBackupInventory[BackupGroup](resp)
+ if err != nil {
+ return nil, err
}
log.Debug().
Str("namespace", namespace).
- Int("count", len(result.Data)).
+ Int("count", len(groups)).
Msg("PBS API: Backup groups found")
- return result.Data, nil
+ return groups, nil
}
// ListBackupSnapshots lists all snapshots for a specific backup group
@@ -2050,24 +2020,9 @@ func (c *Client) ListBackupSnapshots(ctx context.Context, datastore string, name
if err != nil {
return nil, err
}
- defer resp.Body.Close()
-
- if resp.StatusCode != http.StatusOK {
- body, err := readResponseBodyLimited(resp.Body)
- if err != nil {
- return nil, err
- }
- return nil, fmt.Errorf("unexpected status %d: %s", resp.StatusCode, string(body))
- }
-
- var result struct {
- Data []BackupSnapshot `json:"data"`
- }
- if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
- return nil, fmt.Errorf("failed to decode snapshots: %w", err)
- }
- if result.Data == nil {
- return nil, fmt.Errorf("snapshot list response contains no data array")
+ snapshots, err := readBackupInventory[BackupSnapshot](resp)
+ if err != nil {
+ return nil, err
}
// The request selects one exact group. A foreign row or repeated native
@@ -2075,8 +2030,8 @@ func (c *Client) ListBackupSnapshots(ctx context.Context, datastore string, name
// history or an authoritative empty group. Validate before callers sort,
// cap or cache rows; their existing malformed-read path retains old evidence
// without renewing it. Do not put provider payloads in the error.
- seen := make(map[int64]struct{}, len(result.Data))
- for i, snapshot := range result.Data {
+ seen := make(map[int64]struct{}, len(snapshots))
+ for i, snapshot := range snapshots {
if snapshot.BackupType != backupType || snapshot.BackupID != backupID {
return nil, fmt.Errorf("snapshot list response row %d does not match requested backup group", i)
}
@@ -2086,7 +2041,7 @@ func (c *Client) ListBackupSnapshots(ctx context.Context, datastore string, name
seen[snapshot.BackupTime] = struct{}{}
}
- return result.Data, nil
+ return snapshots, nil
}
// ListAllBackups fetches all backups from all namespaces concurrently
diff --git a/pkg/pbs/client_http_test.go b/pkg/pbs/client_http_test.go
index f6023a9c03..f9ae9b4c1a 100644
--- a/pkg/pbs/client_http_test.go
+++ b/pkg/pbs/client_http_test.go
@@ -1030,3 +1030,142 @@ func TestClient_GetNodeStatus_MetricCompleteness(t *testing.T) {
})
}
}
+
+// All three listings carry inventory authority: an accepted empty list may
+// remove previous backup evidence. A decodable prefix is not a complete read.
+func backupInventoryReaders() []struct {
+ name string
+ read func(*Client) (bool, error)
+} {
+ return []struct {
+ name string
+ read func(*Client) (bool, error)
+ }{
+ {"namespaces", func(c *Client) (bool, error) {
+ rows, err := c.ListNamespaces(context.Background(), "archive", "tenant", 2)
+ return rows == nil, err
+ }},
+ {"groups", func(c *Client) (bool, error) {
+ rows, err := c.ListBackupGroups(context.Background(), "archive", "tenant")
+ return rows == nil, err
+ }},
+ {"snapshots", func(c *Client) (bool, error) {
+ rows, err := c.ListBackupSnapshots(context.Background(), "archive", "tenant", "vm", "101")
+ return rows == nil, err
+ }},
+ }
+}
+
+func TestBackupInventoryRequiresWholeResponse(t *testing.T) {
+ bodies := map[string]string{
+ "second value": `{"data":[]} {"data":[]}`,
+ "trailing garbage": `{"data":[]} private-provider-detail`,
+ "reported error": `{"data":[],"errors":{"private":"private-provider-detail"}}`,
+ "empty errors object": `{"data":[],"errors":{}}`,
+ "false errors": `{"data":[],"errors":false}`,
+ "duplicate data": `{"data":null,"data":[]}`,
+ "escaped duplicate data": `{"data":null,"d\u0061ta":[]}`,
+ "duplicate errors": `{"errors":{"private":"private-provider-detail"},"data":[],"errors":null}`,
+ "case data": `{"data":null,"Data":[]}`,
+ "case errors": `{"data":[],"errors":{"private":"private-provider-detail"},"Errors":null}`,
+ "invalid utf8": "{\"data\":[],\"metadata\":\"\xff\"}",
+ }
+ for _, reader := range backupInventoryReaders() {
+ for name, body := range bodies {
+ t.Run(reader.name+"/"+name, func(t *testing.T) {
+ reads := 0
+ client, server := newTestClient(t, func(w http.ResponseWriter, r *http.Request) {
+ reads++
+ if r.Method != http.MethodGet {
+ t.Errorf("inventory attempted %s", r.Method)
+ }
+ _, _ = w.Write([]byte(body))
+ })
+ defer server.Close()
+ isNil, err := reader.read(client)
+ if err == nil || !isNil {
+ t.Fatalf("unread inventory accepted: nil=%t err=%v", isNil, err)
+ }
+ if strings.Contains(err.Error(), "private-provider-detail") {
+ t.Fatal("provider contents leaked in format diagnostic")
+ }
+ if reads != 1 {
+ t.Fatalf("malformed inventory caused %d requests", reads)
+ }
+ })
+ }
+ }
+}
+
+type inventoryResponseBody struct {
+ io.Reader
+ closeErr error
+ closed bool
+ bytes int
+}
+
+func (b *inventoryResponseBody) Read(p []byte) (int, error) {
+ n, err := b.Reader.Read(p)
+ b.bytes += n
+ return n, err
+}
+func (b *inventoryResponseBody) Close() error { b.closed = true; return b.closeErr }
+
+type inventoryReadFailure struct{}
+
+func (inventoryReadFailure) Read([]byte) (int, error) { return 0, errors.New("fixture read failure") }
+
+func TestBackupInventoryRequiresCompleteBody(t *testing.T) {
+ for _, reader := range backupInventoryReaders() {
+ for _, failure := range []string{"read after JSON", "close", "size", "partial status"} {
+ t.Run(reader.name+"/"+failure, func(t *testing.T) {
+ body := &inventoryResponseBody{Reader: strings.NewReader(`{"data":[]}`)}
+ status := http.StatusOK
+ switch failure {
+ case "read after JSON":
+ body.Reader = io.MultiReader(body.Reader, inventoryReadFailure{})
+ case "close":
+ body.closeErr = errors.New("fixture close failure")
+ case "size":
+ body.Reader = io.MultiReader(body.Reader, strings.NewReader(strings.Repeat(" ", int(maxResponseBodyBytes))))
+ case "partial status":
+ status = http.StatusPartialContent
+ }
+ client, err := NewClient(ClientConfig{Host: "https://fixture.invalid", TokenName: "fixture@pbs!read", TokenValue: "fixture-only"})
+ if err != nil {
+ t.Fatal(err)
+ }
+ reads := 0
+ client.httpClient = &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
+ reads++
+ return &http.Response{StatusCode: status, Body: body, Header: make(http.Header)}, nil
+ })}
+ isNil, err := reader.read(client)
+ if err == nil || !isNil {
+ t.Errorf("incomplete inventory accepted: nil=%t err=%v", isNil, err)
+ }
+ if reads != 1 || !body.closed {
+ t.Errorf("reads=%d closed=%t", reads, body.closed)
+ }
+ if int64(body.bytes) > maxResponseBodyBytes+1 {
+ t.Errorf("read %d bytes past body bound", body.bytes)
+ }
+ })
+ }
+ }
+}
+
+func TestBackupInventoryAcceptsCompleteEmptyResponse(t *testing.T) {
+ for _, reader := range backupInventoryReaders() {
+ for _, body := range []string{`{"data":[]}`, ` {"data":[],"errors":null,"total":0,"metadata":{"future":"温度"}} `} {
+ t.Run(reader.name+"/"+body, func(t *testing.T) {
+ client, server := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte(body)) })
+ defer server.Close()
+ isNil, err := reader.read(client)
+ if err != nil || isNil {
+ t.Fatalf("complete empty read rejected: nil=%t err=%v", isNil, err)
+ }
+ })
+ }
+ }
+}