Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Require whole PBS inventory responses before reconciling history
pulse-triage[bot]0d2aa2a731aa
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex 189c5a53f6..bd14594ef4 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -17,6 +17,26 @@## Purpose+### PBS backup listings require the whole response++Namespace, group and snapshot reads accept one unambiguous JSON envelope only+from a fully read and successfully closed HTTP 200 body within the existing+8 MiB bound. Trailing values/garbage, invalid UTF-8, duplicate envelope fields,+case aliases of `data`/`errors`, and non-null error envelopes are unread, not+accepted empty inventory or fresh protection evidence. Unique future metadata+and explicit empty arrays remain valid; the older-PBS namespace 404 fallback is+unchanged. Diagnostics contain no provider payload. No retry, auth, poll budget,+identity, guest operation or retention policy changes.++Normal client suites cover body/close/bound failures and envelope controls.+`TestPBSWholeInventoryResponseRetainsEvidence` exercises VM/CT cold/warm HTTP+collection through live state, cache times, durable points and posture: unread+scopes retain old evidence without renewing it, readable siblings advance, and+ordinary complete/empty reads recover and reconcile absence.+`TestPBSWholeInventoryConsumerLifecycle` feeds those actual poll results to+source-bound desktop/narrow Backups inspection. This is truthful collection,+not original guest recovery, an established #3248 cause or release availability.+### Unread PBS writer inventories do not establish a terminal backupThe running backup/sync task reader requires an explicit `data` array from adiff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.mdindex 2011e16cf4..877da1baef 100644--- a/docs/release-control/v6/internal/subsystems/storage-recovery.md+++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md@@ -21,6 +21,26 @@## Purpose+### PBS backup listings require the whole response++Namespace, group and snapshot reads accept one unambiguous JSON envelope only+from a fully read and successfully closed HTTP 200 body within the existing+8 MiB bound. Trailing values/garbage, invalid UTF-8, duplicate envelope fields,+case aliases of `data`/`errors`, and non-null error envelopes are unread, not+accepted empty inventory or fresh protection evidence. Unique future metadata+and explicit empty arrays remain valid; the older-PBS namespace 404 fallback is+unchanged. Diagnostics contain no provider payload. No retry, auth, poll budget,+identity, guest operation or retention policy changes.++Normal client suites cover body/close/bound failures and envelope controls.+`TestPBSWholeInventoryResponseRetainsEvidence` exercises VM/CT cold/warm HTTP+collection through live state, cache times, durable points and posture: unread+scopes retain old evidence without renewing it, readable siblings advance, and+ordinary complete/empty reads recover and reconcile absence.+`TestPBSWholeInventoryConsumerLifecycle` feeds those actual poll results to+source-bound desktop/narrow Backups inspection. This is truthful collection,+not original guest recovery, an established #3248 cause or release availability.+### Unread PBS writer inventories do not establish a terminal backupThe running backup/sync task reader requires an explicit `data` array from adiff --git a/internal/monitoring/pbs_inventory_response_test.go b/internal/monitoring/pbs_inventory_response_test.gonew file mode 100644index 0000000000..1d29d37c3a--- /dev/null+++ b/internal/monitoring/pbs_inventory_response_test.go@@ -0,0 +1,169 @@+package monitoring++import (+ "crypto/sha256"+ "encoding/json"+ "fmt"+ "os"+ "reflect"+ "testing"++ "github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust"+ "github.com/rcourtman/pulse-go-rewrite/internal/recovery"+)++func TestPBSWholeInventoryResponseRetainsEvidence(t *testing.T) {+ for _, kind := range []string{"vm", "ct"} {+ for _, stage := range []string{"namespace", "groups", "snapshots"} {+ for name, body := range map[string]string{+ "trailing value": `{"data":[]} {"data":[]}`,+ "error envelope": `{"data":[],"errors":{"private":"fixture-only"}}`,+ "overwritten data": `{"data":null,"data":[]}`,+ "invalid utf8": "{\"data\":[],\"metadata\":\"\xff\"}",+ } {+ for _, warm := range []bool{false, true} {+ t.Run(fmt.Sprintf("%s/%s/%s/warm=%t", kind, stage, name, warm), func(t *testing.T) {+ f := newPBSNamespaceContinuityFixture(t, kind)+ if warm {+ f.collect(t, true, nil)+ }+ before, points := f.live(), f.points(t, "pbs-main")+ key := pbsBackupGroupKey{datastore: "archive", namespace: "tenant", backupType: kind, backupID: "101"}+ cacheAt := f.m.pbsBackupCacheTimeFor("pbs-main", key)+ f.rootAt.Add(15 * 60)+ f.tenantAt.Add(15 * 60)+ for range 2 {+ f.collectInventoryResponse(t, stage, body)+ rows := f.live()+ want := 1+ if warm {+ want = 2+ if !reflect.DeepEqual(rows["101"], before["101"]) {+ t.Error("unread response removed or changed accepted live file")+ }+ }+ if len(rows) != want || rows["100"].BackupTime.Unix() != f.rootAt.Load() {+ t.Error("unread response invented/lost files or held back readable sibling")+ }+ if !f.m.pbsBackupCacheTimeFor("pbs-main", key).Equal(cacheAt) {+ t.Error("unread response renewed snapshot cache time")+ }+ for _, prior := range points {+ if prior.Details["namespace"] != "tenant" {+ continue+ }+ found := false+ for _, now := range f.points(t, "pbs-main") {+ found = found || (prior.ID == now.ID && reflect.DeepEqual(prior, now))+ }+ if !found {+ t.Error("unread response removed or renewed durable evidence")+ }+ }+ for _, p := range f.postures(t) {+ if !warm && p.SubjectResourceID == f.ids[1] {+ if p.State != recovery.ProtectionStateUnknown || len(p.ProviderStates) != 0 {+ t.Error("cold unread scope acquired evidence")+ }+ continue+ }+ if p.State == recovery.ProtectionStateProtected || len(p.ProviderStates) != 1 || p.ProviderStates[0].HistoryCompleteness != recovery.ProtectionHistoryPartial || p.ProviderStates[0].Permissions != operationaltrust.EvidencePermissionsUnknown {+ t.Errorf("unread response claimed protection or denied access: %+v", p)+ }+ }+ }+ if f.failedReads.Load() != 2 {+ t.Error("invalid response caused extra reads")+ }+ f.inventoryResponse.Store(nil)+ f.collect(t, true, nil)+ if f.live()["101"].BackupTime.Unix() != f.tenantAt.Load() {+ t.Error("complete ordinary read failed to recover")+ }+ for _, p := range f.postures(t) {+ if p.State != recovery.ProtectionStateProtected {+ t.Error("ordinary recovery failed to establish complete protection")+ }+ }+ f.rootEmpty.Store(true)+ f.tenantEmpty.Store(true)+ f.collect(t, true, nil)+ if len(f.live()) != 0 || len(f.points(t, "pbs-main")) != 0 {+ t.Error("complete empty read failed to reconcile confirmed absence")+ }+ })+ }+ }+ }+ }+}++// Exercise ordinary polling, raw publication, durable ingestion and posture.+// Optional export is evidence for the real Backups consumer, not a second path.+func TestPBSWholeInventoryConsumerLifecycle(t *testing.T) {+ for _, kind := range []string{"vm", "ct"} {+ t.Run(kind, func(t *testing.T) {+ f := newPBSNamespaceContinuityFixture(t, kind)+ frames := map[string]any{}+ for _, phase := range []string{"observed", "unread-namespaces", "unread-groups", "unread-snapshots", "recovered", "confirmed-empty"} {+ switch phase {+ case "unread-namespaces":+ f.rootAt.Add(15 * 60)+ f.collectInventoryResponse(t, "namespace", `{"data":[]} {"data":[]}`)+ case "unread-groups":+ f.collectInventoryResponse(t, "groups", `{"data":[],"errors":{"private":"fixture-only"}}`)+ case "unread-snapshots":+ f.tenantAt.Add(15 * 60)+ f.collectInventoryResponse(t, "snapshots", `{"data":null,"data":[]}`)+ default:+ f.inventoryResponse.Store(nil)+ if phase == "confirmed-empty" {+ f.rootEmpty.Store(true)+ f.tenantEmpty.Store(true)+ }+ f.collect(t, true, nil)+ }+ for _, p := range f.postures(t) {+ if p.SubjectResourceID == f.ids[1] || p.SubjectResourceID == f.ids[0] {+ if (phase == "observed" || phase == "recovered") != (p.State == recovery.ProtectionStateProtected) {+ t.Fatalf("phase %s posture=%+v", phase, p)+ }+ }+ }+ want := 2+ wantPoints := 2+ if phase == "confirmed-empty" {+ want = 0+ wantPoints = 0+ } else if phase == "unread-namespaces" || phase == "unread-groups" || phase == "unread-snapshots" {+ wantPoints = 3+ }+ if len(f.live()) != want || len(f.points(t, "pbs-main")) != wantPoints {+ t.Fatalf("phase %s lost or invented inventory", phase)+ }+ frames[phase] = map[string]any{+ "pbs": map[string]any{"data": map[string]any{"backups": f.m.state.GetSnapshot().PBSBackups}},+ "postures": map[string]any{"data": f.postures(t), "policy": recovery.DefaultProtectionPosturePolicy.Payload()},+ "points": f.points(t, "pbs-main"),+ }+ }+ if dest := os.Getenv("PULSE_PBS_RESPONSE_FRAMES"); dest != "" {+ hashes := map[string]string{}+ for _, file := range []string{"monitor_backups.go", "backup_guard.go", "../../pkg/pbs/client.go", "../../pkg/pbs/backup_inventory_response.go"} {+ content, err := os.ReadFile(file)+ if err != nil {+ t.Fatal(err)+ }+ hashes[file] = fmt.Sprintf("%x", sha256.Sum256(content))+ }+ data, err := json.Marshal(map[string]any{"runtime_sha256": hashes, "resource_ids": f.ids, "frames": frames})+ if err != nil {+ t.Fatal(err)+ }+ if err := os.WriteFile(dest+"/frames-"+kind+".json", data, 0600); err != nil {+ t.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/pbs/backup_inventory_response.go b/pkg/pbs/backup_inventory_response.gonew file mode 100644index 0000000000..6017939bcc--- /dev/null+++ b/pkg/pbs/backup_inventory_response.go@@ -0,0 +1,72 @@+package pbs++import (+ "bytes"+ "encoding/json"+ "errors"+ "io"+ "net/http"+ "strings"+ "unicode/utf8"+)++var errInvalidBackupInventoryResponse = errors.New("invalid PBS backup inventory response")++// A list can reconcile absence or renew backup protection. Accept only one+// unambiguous envelope from a fully read, bounded response, never a JSON prefix.+// Keep provider payloads out of format diagnostics (including reported errors).+func readBackupInventory[T any](resp *http.Response) ([]T, error) {+ body, readErr := readResponseBodyLimited(resp.Body)+ closeErr := resp.Body.Close()+ if readErr != nil || closeErr != nil {+ return nil, errors.Join(readErr, closeErr)+ }+ if resp.StatusCode != http.StatusOK || !utf8.Valid(body) {+ return nil, errInvalidBackupInventoryResponse+ }+ decoder := json.NewDecoder(bytes.NewReader(body))+ start, err := decoder.Token()+ if err != nil || start != json.Delim('{') {+ return nil, errInvalidBackupInventoryResponse+ }+ fields := make(map[string]json.RawMessage)+ for decoder.More() {+ token, err := decoder.Token()+ if err != nil {+ return nil, errInvalidBackupInventoryResponse+ }+ name, ok := token.(string)+ if !ok {+ return nil, errInvalidBackupInventoryResponse+ }+ if _, duplicate := fields[name]; duplicate ||+ (name != "data" && strings.EqualFold(name, "data")) ||+ (name != "errors" && strings.EqualFold(name, "errors")) {+ return nil, errInvalidBackupInventoryResponse+ }+ var value json.RawMessage+ if decoder.Decode(&value) != nil {+ return nil, errInvalidBackupInventoryResponse+ }+ fields[name] = value+ }+ end, err := decoder.Token()+ if err != nil || end != json.Delim('}') {+ return nil, errInvalidBackupInventoryResponse+ }+ if _, err := decoder.Token(); err != io.EOF {+ return nil, errInvalidBackupInventoryResponse+ }+ if failure, present := fields["errors"]; present && !bytes.Equal(bytes.TrimSpace(failure), []byte("null")) {+ return nil, errInvalidBackupInventoryResponse+ }+ data := bytes.TrimSpace(fields["data"])+ if len(data) == 0 || data[0] != '[' {+ return nil, errInvalidBackupInventoryResponse+ }+ var rows []T+ if json.Unmarshal(data, &rows) != nil {+ return nil, errInvalidBackupInventoryResponse+ }+ return rows, nil+}diff --git a/pkg/pbs/client.go b/pkg/pbs/client.goindex 24a0c61ed4..894c742144 100644--- a/pkg/pbs/client.go+++ b/pkg/pbs/client.go@@ -1966,22 +1966,7 @@ func (c *Client) ListNamespaces(ctx context.Context, datastore string, parentNam}return nil, err}- defer resp.Body.Close()-- var result struct {- Data []Namespace `json:"data"`- }-- if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {- return nil, err- }- // Match datastore discovery: missing/null data is unread, not an empty- // scope that can authorize removal of retained backup history.- if result.Data == nil {- return nil, fmt.Errorf("namespace list response contains no data array")- }-- return result.Data, nil+ return readBackupInventory[Namespace](resp)}// ListBackupGroups lists all backup groups in a datastore/namespace@@ -2005,31 +1990,16 @@ func (c *Client) ListBackupGroups(ctx context.Context, datastore string, namespaif err != nil {return nil, err}- defer resp.Body.Close()-- if resp.StatusCode != http.StatusOK {- body, err := readResponseBodyLimited(resp.Body)- if err != nil {- return nil, err- }- return nil, fmt.Errorf("unexpected status %d: %s", resp.StatusCode, string(body))- }-- var result struct {- Data []BackupGroup `json:"data"`- }- if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {- return nil, fmt.Errorf("failed to decode backup groups: %w", err)- }- if result.Data == nil {- return nil, fmt.Errorf("backup group list response contains no data array")+ groups, err := readBackupInventory[BackupGroup](resp)+ if err != nil {+ return nil, err}log.Debug().Str("namespace", namespace).- Int("count", len(result.Data)).+ Int("count", len(groups)).Msg("PBS API: Backup groups found")- return result.Data, nil+ return groups, nil}// ListBackupSnapshots lists all snapshots for a specific backup group@@ -2050,24 +2020,9 @@ func (c *Client) ListBackupSnapshots(ctx context.Context, datastore string, nameif err != nil {return nil, err}- defer resp.Body.Close()-- if resp.StatusCode != http.StatusOK {- body, err := readResponseBodyLimited(resp.Body)- if err != nil {- return nil, err- }- return nil, fmt.Errorf("unexpected status %d: %s", resp.StatusCode, string(body))- }-- var result struct {- Data []BackupSnapshot `json:"data"`- }- if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {- return nil, fmt.Errorf("failed to decode snapshots: %w", err)- }- if result.Data == nil {- return nil, fmt.Errorf("snapshot list response contains no data array")+ snapshots, err := readBackupInventory[BackupSnapshot](resp)+ if err != nil {+ return nil, err}// The request selects one exact group. A foreign row or repeated native@@ -2075,8 +2030,8 @@ func (c *Client) ListBackupSnapshots(ctx context.Context, datastore string, name// history or an authoritative empty group. Validate before callers sort,// cap or cache rows; their existing malformed-read path retains old evidence// without renewing it. Do not put provider payloads in the error.- seen := make(map[int64]struct{}, len(result.Data))- for i, snapshot := range result.Data {+ seen := make(map[int64]struct{}, len(snapshots))+ for i, snapshot := range snapshots {if snapshot.BackupType != backupType || snapshot.BackupID != backupID {return nil, fmt.Errorf("snapshot list response row %d does not match requested backup group", i)}@@ -2086,7 +2041,7 @@ func (c *Client) ListBackupSnapshots(ctx context.Context, datastore string, nameseen[snapshot.BackupTime] = struct{}{}}- return result.Data, nil+ return snapshots, nil}// ListAllBackups fetches all backups from all namespaces concurrentlydiff --git a/pkg/pbs/client_http_test.go b/pkg/pbs/client_http_test.goindex f6023a9c03..f9ae9b4c1a 100644--- a/pkg/pbs/client_http_test.go+++ b/pkg/pbs/client_http_test.go@@ -1030,3 +1030,142 @@ func TestClient_GetNodeStatus_MetricCompleteness(t *testing.T) {})}}++// All three listings carry inventory authority: an accepted empty list may+// remove previous backup evidence. A decodable prefix is not a complete read.+func backupInventoryReaders() []struct {+ name string+ read func(*Client) (bool, error)+} {+ return []struct {+ name string+ read func(*Client) (bool, error)+ }{+ {"namespaces", func(c *Client) (bool, error) {+ rows, err := c.ListNamespaces(context.Background(), "archive", "tenant", 2)+ return rows == nil, err+ }},+ {"groups", func(c *Client) (bool, error) {+ rows, err := c.ListBackupGroups(context.Background(), "archive", "tenant")+ return rows == nil, err+ }},+ {"snapshots", func(c *Client) (bool, error) {+ rows, err := c.ListBackupSnapshots(context.Background(), "archive", "tenant", "vm", "101")+ return rows == nil, err+ }},+ }+}++func TestBackupInventoryRequiresWholeResponse(t *testing.T) {+ bodies := map[string]string{+ "second value": `{"data":[]} {"data":[]}`,+ "trailing garbage": `{"data":[]} private-provider-detail`,+ "reported error": `{"data":[],"errors":{"private":"private-provider-detail"}}`,+ "empty errors object": `{"data":[],"errors":{}}`,+ "false errors": `{"data":[],"errors":false}`,+ "duplicate data": `{"data":null,"data":[]}`,+ "escaped duplicate data": `{"data":null,"d\u0061ta":[]}`,+ "duplicate errors": `{"errors":{"private":"private-provider-detail"},"data":[],"errors":null}`,+ "case data": `{"data":null,"Data":[]}`,+ "case errors": `{"data":[],"errors":{"private":"private-provider-detail"},"Errors":null}`,+ "invalid utf8": "{\"data\":[],\"metadata\":\"\xff\"}",+ }+ for _, reader := range backupInventoryReaders() {+ for name, body := range bodies {+ t.Run(reader.name+"/"+name, func(t *testing.T) {+ reads := 0+ client, server := newTestClient(t, func(w http.ResponseWriter, r *http.Request) {+ reads+++ if r.Method != http.MethodGet {+ t.Errorf("inventory attempted %s", r.Method)+ }+ _, _ = w.Write([]byte(body))+ })+ defer server.Close()+ isNil, err := reader.read(client)+ if err == nil || !isNil {+ t.Fatalf("unread inventory accepted: nil=%t err=%v", isNil, err)+ }+ if strings.Contains(err.Error(), "private-provider-detail") {+ t.Fatal("provider contents leaked in format diagnostic")+ }+ if reads != 1 {+ t.Fatalf("malformed inventory caused %d requests", reads)+ }+ })+ }+ }+}++type inventoryResponseBody struct {+ io.Reader+ closeErr error+ closed bool+ bytes int+}++func (b *inventoryResponseBody) Read(p []byte) (int, error) {+ n, err := b.Reader.Read(p)+ b.bytes += n+ return n, err+}+func (b *inventoryResponseBody) Close() error { b.closed = true; return b.closeErr }++type inventoryReadFailure struct{}++func (inventoryReadFailure) Read([]byte) (int, error) { return 0, errors.New("fixture read failure") }++func TestBackupInventoryRequiresCompleteBody(t *testing.T) {+ for _, reader := range backupInventoryReaders() {+ for _, failure := range []string{"read after JSON", "close", "size", "partial status"} {+ t.Run(reader.name+"/"+failure, func(t *testing.T) {+ body := &inventoryResponseBody{Reader: strings.NewReader(`{"data":[]}`)}+ status := http.StatusOK+ switch failure {+ case "read after JSON":+ body.Reader = io.MultiReader(body.Reader, inventoryReadFailure{})+ case "close":+ body.closeErr = errors.New("fixture close failure")+ case "size":+ body.Reader = io.MultiReader(body.Reader, strings.NewReader(strings.Repeat(" ", int(maxResponseBodyBytes))))+ case "partial status":+ status = http.StatusPartialContent+ }+ client, err := NewClient(ClientConfig{Host: "https://fixture.invalid", TokenName: "fixture@pbs!read", TokenValue: "fixture-only"})+ if err != nil {+ t.Fatal(err)+ }+ reads := 0+ client.httpClient = &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {+ reads+++ return &http.Response{StatusCode: status, Body: body, Header: make(http.Header)}, nil+ })}+ isNil, err := reader.read(client)+ if err == nil || !isNil {+ t.Errorf("incomplete inventory accepted: nil=%t err=%v", isNil, err)+ }+ if reads != 1 || !body.closed {+ t.Errorf("reads=%d closed=%t", reads, body.closed)+ }+ if int64(body.bytes) > maxResponseBodyBytes+1 {+ t.Errorf("read %d bytes past body bound", body.bytes)+ }+ })+ }+ }+}++func TestBackupInventoryAcceptsCompleteEmptyResponse(t *testing.T) {+ for _, reader := range backupInventoryReaders() {+ for _, body := range []string{`{"data":[]}`, ` {"data":[],"errors":null,"total":0,"metadata":{"future":"温度"}} `} {+ t.Run(reader.name+"/"+body, func(t *testing.T) {+ client, server := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte(body)) })+ defer server.Close()+ isNil, err := reader.read(client)+ if err != nil || isNil {+ t.Fatalf("complete empty read rejected: nil=%t err=%v", isNil, err)+ }+ })+ }+ }+}