Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Reject incomplete Proxmox storage inventories
pulse-triage[bot]ce1fc7e5ff9c
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex 7d3aa81132..62197dd277 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -8389,3 +8389,26 @@ complete recovery and confirmed scoped removal. `TestPVESnapshotHTTPBrowserProdufeeds the actual consuming proof. These are synthetic HTTP/provider fixtures,not native guest safety, backup execution, installed recovery or a diagnosis of#3248. The separate prior lifecycle-suite failure remains unexplained.++### Storage HTTP success must be a complete inventory++PVE node storage, cluster storage configuration and storage content readers admit+only a fully read, bounded HTTP-200 response with one unambiguous JSON object and+an explicit `data` array. Missing/null inventories, error-bearing or conflicting+envelopes, trailing bytes, invalid rows and duplicate identities are unread+history, not successful empty or partial enumerations. Every content row is+validated before filtering backup/template kinds. Node catalogs require their+storage identity, type and supported-content discriminators; configuration-only+`/storage` rows retain optional/default fields. Optional metadata, host backups,+zero-byte/in-progress entries and explicit empty arrays remain supported. No+schema, transport, auth, failover, query admission or polling budget changes.++`TestStorageInventoryResponseAdmission`, accepted-field/filter controls and+completed-response tests exercise all three HTTP readers. The registered+`TestPVEStorageHTTP*` regressions connect real HTTP reads to storage backup+polling, asynchronous ingest and canonical protection: warm/cold history,+independently accepted peers, all-unread state, repeated failures, ordinary+recovery and confirmed scoped absence. Their opt-in producer feeds the existing+production Backups consumer for browser proof. Retained evidence is not a fresh+read, available archive, installed recovery or a diagnosis of #3248; no guest or+backup operation, thaw or filesystem-liveness assertion follows.diff --git a/internal/monitoring/storage_inventory_http_test.go b/internal/monitoring/storage_inventory_http_test.gonew file mode 100644index 0000000000..5034d12b69--- /dev/null+++ b/internal/monitoring/storage_inventory_http_test.go@@ -0,0 +1,299 @@+package monitoring++import (+ "context"+ "encoding/json"+ "fmt"+ "net/http"+ "net/http/httptest"+ "os"+ "reflect"+ "strings"+ "sync"+ "testing"+ "time"++ "github.com/rcourtman/pulse-go-rewrite/internal/models"+ "github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust"+ "github.com/rcourtman/pulse-go-rewrite/internal/recovery"+ "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"+)++// Only storage catalog/content reads go through the real HTTP client. No guest+// commands, backup operation, external service or database-internal change.+type storageInventoryHTTPClient struct {+ *pveReconciliationClient+ api *proxmox.Client+}++func (c *storageInventoryHTTPClient) GetStorage(ctx context.Context, node string) ([]proxmox.Storage, error) {+ return c.api.GetStorage(ctx, node)+}++func (c *storageInventoryHTTPClient) GetStorageContent(ctx context.Context, node, storage string) ([]proxmox.StorageContent, error) {+ return c.api.GetStorageContent(ctx, node, storage)+}++type storageInventoryHTTPFixture struct {+ f *pveReconciliationFixture+ client *storageInventoryHTTPClient+ mu sync.Mutex+ bodies map[string]string+ paths []string+}++func storageInventoryHTTPPath(kind, node string) string {+ path := "/api2/json/nodes/" + node + "/storage"+ if kind == "content" {+ path += "/archive-" + node + "/content"+ }+ return path+}++func newStorageInventoryHTTPFixture(t *testing.T) *storageInventoryHTTPFixture {+ t.Helper()+ h := &storageInventoryHTTPFixture{f: newPVEReconciliationFixture(t), bodies: make(map[string]string)}+ // Normal storage inventory supplies the carry-forward lookup when a node's+ // catalog cannot be read. Cold-state cases deliberately discard that cache.+ h.f.m.state.UpdateStorage([]models.Storage{+ {ID: "cluster-node-a-archive-node-a", Name: "archive-node-a", Node: "node-a", Instance: "cluster", Content: "backup", Type: "dir"},+ {ID: "cluster-node-b-archive-node-b", Name: "archive-node-b", Node: "node-b", Instance: "cluster", Content: "backup", Type: "dir"},+ })+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ h.mu.Lock()+ defer h.mu.Unlock()+ h.paths = append(h.paths, r.URL.Path)+ w.Header().Set("Content-Type", "application/json")+ if body, ok := h.bodies[r.URL.Path]; ok {+ fmt.Fprint(w, body)+ return+ }+ for _, node := range []string{"node-a", "node-b"} {+ var data any+ if r.URL.Path == storageInventoryHTTPPath("catalog", node) {+ data, _ = h.f.client.GetStorage(r.Context(), node)+ } else if r.URL.Path == storageInventoryHTTPPath("content", node) {+ data, _ = h.f.client.GetStorageContent(r.Context(), node, "archive-"+node)+ } else {+ continue+ }+ if err := json.NewEncoder(w).Encode(map[string]any{"data": data}); err != nil {+ t.Error(err)+ }+ return+ }+ t.Errorf("unexpected fixture request: %s", r.URL.Path)+ http.NotFound(w, r)+ }))+ t.Cleanup(server.Close)+ api, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: 5 * time.Second})+ if err != nil {+ t.Fatal(err)+ }+ h.client = &storageInventoryHTTPClient{pveReconciliationClient: h.f.client, api: api}+ return h+}++func (h *storageInventoryHTTPFixture) set(kind, node, body string) {+ h.mu.Lock()+ defer h.mu.Unlock()+ if body == "" {+ delete(h.bodies, storageInventoryHTTPPath(kind, node))+ } else {+ h.bodies[storageInventoryHTTPPath(kind, node)] = body+ }+}++func (h *storageInventoryHTTPFixture) advance() {+ h.mu.Lock()+ defer h.mu.Unlock()+ h.f.client.at = h.f.client.at.Add(15 * time.Minute)+}++func (h *storageInventoryHTTPFixture) poll(t *testing.T, wantReads int) {+ t.Helper()+ h.mu.Lock()+ before := len(h.paths)+ h.mu.Unlock()+ h.f.m.pollStorageBackupsWithNodes(context.Background(), "cluster", h.client, h.f.nodes, h.f.online)+ h.f.wait(t)+ h.mu.Lock()+ defer h.mu.Unlock()+ if len(h.paths)-before != wantReads {+ t.Fatalf("got HTTP reads %v, want %d; no retries or other operations", h.paths[before:], wantReads)+ }+}++func TestPVEStorageHTTPInvalidResponseRetainsHistory(t *testing.T) {+ for _, kind := range []string{"catalog", "content"} {+ row := `{"storage":"archive-node-b","type":"dir","content":"backup","enabled":1,"active":1}`+ missingDiscriminator := `{"storage":"archive-node-b","type":"dir"}`+ if kind == "content" {+ row = `{"volid":"archive-node-b:backup/vzdump-qemu-102.vma.zst","vmid":102,"content":"backup","ctime":1700000000}`+ missingDiscriminator = `{"volid":"archive-node-b:backup/vzdump-qemu-102.vma.zst"}`+ }+ cases := map[string]string{+ "missing-data": `{}`,+ "null-data": `{"data":null}`,+ "conflicting-error": `{"data":[],"errors":{"message":"fixture-private-detail"}}`,+ "duplicate-data": `{"data":[` + row + `],"data":[]}`,+ "trailing-value": `{"data":[]} {"data":[]}`,+ "null-row": `{"data":[null]}`,+ "bad-row-after-good": `{"data":[` + row + `,null]}`,+ "missing-discriminator": `{"data":[` + missingDiscriminator + `]}`,+ }+ for _, cold := range []bool{false, true} {+ for name, body := range cases {+ t.Run(fmt.Sprintf("%s/cold=%t/%s", kind, cold, name), func(t *testing.T) {+ h := newStorageInventoryHTTPFixture(t)+ f := h.f+ h.poll(t, 4)+ other := recoveryIngestTestPoint("pve-backup:unrelated", "other-site", f.client.at)+ if err := f.store.UpsertPoints(context.Background(), []recovery.RecoveryPoint{other}); err != nil {+ t.Fatal(err)+ }+ before := f.points(t)+ if len(before) != 3 {+ t.Fatalf("healthy control points=%d, want 3", len(before))+ }+ beforeRows := f.m.state.GetSnapshot().PVEBackups.StorageBackups+ if cold {+ f.coldState()+ }+ h.advance()+ h.set(kind, "node-b", body)+ reads := 4+ if kind == "catalog" {+ reads = 3+ }+ for range 2 {+ h.poll(t, reads)+ after := f.points(t)+ if len(after) != 3 || !reflect.DeepEqual(before[other.ID], after[other.ID]) {+ t.Fatal("malformed success deleted history, admitted extra rows or changed another instance")+ }+ for id, point := range before {+ if point.SubjectResourceID == f.ids[1] && !reflect.DeepEqual(point, after[id]) {+ t.Fatal("unreadable backup was removed or its observation renewed")+ }+ if point.SubjectResourceID == f.ids[0] && (after[id].CompletedAt == nil || !after[id].CompletedAt.Equal(f.client.at)) {+ t.Fatal("independent accepted storage did not update")+ }+ }+ f.assertHistory(t, recovery.ProtectionHistoryPartial, operationaltrust.EvidencePermissionsUnknown)+ rows := f.m.state.GetSnapshot().PVEBackups.StorageBackups+ want := 2+ if cold {+ want = 1+ }+ if len(rows) != want {+ t.Fatalf("live rows=%d want=%d; unread history cannot erase warm rows or invent cold ones", len(rows), want)+ }+ if !cold {+ for _, row := range rows {+ if row.VMID == 102 && !reflect.DeepEqual(row, beforeRows[1]) {+ t.Fatal("unread HTTP inventory changed its warm row")+ }+ }+ }+ }+ h.set(kind, "node-b", "")+ h.poll(t, 4)+ f.assertHistory(t, recovery.ProtectionHistoryComplete, operationaltrust.EvidencePermissionsSufficient)+ if len(f.m.state.GetSnapshot().PVEBackups.StorageBackups) != 2 {+ t.Fatal("accepted recovery did not restore live inventory")+ }+ // Explicit scoped absence still clears only this connection.+ h.set(kind, "node-a", `{"data":[]}`)+ h.set(kind, "node-b", `{"data":[]}`)+ reads = 4+ if kind == "catalog" {+ reads = 2+ }+ h.poll(t, reads)+ if points := f.points(t); len(points) != 1 || !reflect.DeepEqual(before[other.ID], points[other.ID]) || len(f.m.state.GetSnapshot().PVEBackups.StorageBackups) != 0 {+ t.Fatal("explicit empty response failed to clear exactly its own backup scope")+ }+ })+ }+ }+ }+}++func TestPVEStorageHTTPAllUnreadCannotClaimEmpty(t *testing.T) {+ for _, kind := range []string{"catalog", "content"} {+ for _, cold := range []bool{false, true} {+ t.Run(fmt.Sprintf("%s/cold=%t", kind, cold), func(t *testing.T) {+ h := newStorageInventoryHTTPFixture(t)+ h.poll(t, 4)+ before := h.f.points(t)+ if cold {+ h.f.coldState()+ }+ for _, node := range []string{"node-a", "node-b"} {+ h.set(kind, node, `{"data":null}`)+ }+ reads := 4+ if kind == "catalog" {+ reads = 2+ }+ for range 2 {+ h.poll(t, reads)+ if !reflect.DeepEqual(before, h.f.points(t)) {+ t.Fatal("all-unread response removed or renewed recovery evidence")+ }+ h.f.assertHistory(t, recovery.ProtectionHistoryUnavailable, operationaltrust.EvidencePermissionsUnknown)+ }+ })+ }+ }+}++// The browser consumes these actual HTTP -> poll -> queue -> store -> posture+// frames, not a separately invented provider state. Export stays opt-in.+func TestPVEStorageHTTPBrowserProducer(t *testing.T) {+ if os.Getenv("PULSE_STORAGE_HTTP_FRAMES") != "1" {+ t.Skip("producer export is opt-in")+ }+ h := newStorageInventoryHTTPFixture(t)+ frames := make(map[string]any)+ for _, phase := range []string{"observed", "catalog-unread", "content-unread", "cold-unread", "recovered", "confirmed-empty"} {+ switch phase {+ case "catalog-unread":+ h.set("catalog", "node-b", `{}`)+ h.advance()+ case "content-unread":+ h.set("catalog", "node-b", "")+ h.set("content", "node-b", `{"data":[{"volid":"unaccepted","content":"backup"},null]}`)+ case "cold-unread":+ h.f.coldState()+ case "recovered":+ h.set("content", "node-b", "")+ case "confirmed-empty":+ h.set("content", "node-a", `{"data":[]}`)+ h.set("content", "node-b", `{"data":[]}`)+ }+ reads := 4+ if phase == "catalog-unread" {+ reads = 3+ }+ h.poll(t, reads)+ points := h.f.points(t)+ want := 2+ if phase == "confirmed-empty" {+ want = 0+ }+ if len(points) != want {+ t.Fatalf("phase %s points=%d want=%d", phase, len(points), want)+ }+ frames[phase] = map[string]any{"pve": map[string]any{"data": h.f.m.state.GetSnapshot().PVEBackups},+ "postures": map[string]any{"data": h.f.postures(t), "policy": recovery.DefaultProtectionPosturePolicy.Payload()}, "points": points}+ }+ data, err := json.Marshal(map[string]any{"observed_at": time.Now().UTC(), "resource_ids": h.f.ids, "frames": frames, "paths": h.paths})+ if err != nil {+ t.Fatal(err)+ }+ // One Go-JSON event may hold only part of this line; collectors must join+ // all Output events before extracting the producer frame.+ t.Log("STORAGE_HTTP_FRAMES=" + strings.TrimSpace(string(data)))+}diff --git a/pkg/proxmox/client.go b/pkg/proxmox/client.goindex eb342ba1a7..c3b27c202c 100644--- a/pkg/proxmox/client.go+++ b/pkg/proxmox/client.go@@ -1357,17 +1357,7 @@ func (c *Client) GetStorage(ctx context.Context, node string) ([]Storage, error)if err != nil {return nil, err}- defer resp.Body.Close()-- var result struct {- Data []Storage `json:"data"`- }-- if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {- return nil, err- }-- return result.Data, nil+ return readStorageCatalog(resp, true)}// GetAllStorage returns storage information across all nodes@@ -1387,17 +1377,7 @@ func (c *Client) GetAllStorage(ctx context.Context) ([]Storage, error) {if err != nil {return nil, err}- defer resp.Body.Close()-- var result struct {- Data []Storage `json:"data"`- }-- if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {- return nil, err- }-- return result.Data, nil+ return readStorageCatalog(resp, false)}// Task represents a Proxmox task@@ -1561,19 +1541,14 @@ func (c *Client) GetStorageContent(ctx context.Context, node, storage string) ([if err != nil {return nil, err}- defer resp.Body.Close()-- var result struct {- Data []StorageContent `json:"data"`- }-- if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {+ contents, err := readStorageContentInventory(resp)+ if err != nil {return nil, err}// Filter for backup content onlyvar backups []StorageContent- for _, content := range result.Data {+ for _, content := range contents {if content.Content == "backup" || content.Content == "vztmpl" {backups = append(backups, content)}diff --git a/pkg/proxmox/client_api_more4_test.go b/pkg/proxmox/client_api_more4_test.goindex f46797b087..0f0fd5cd48 100644--- a/pkg/proxmox/client_api_more4_test.go+++ b/pkg/proxmox/client_api_more4_test.go@@ -132,7 +132,7 @@ func TestClient_GetContainers(t *testing.T) {func TestClient_GetStorage(t *testing.T) {server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {w.Header().Set("Content-Type", "application/json")- fmt.Fprint(w, `{"data":[{"storage":"local","type":"dir","total":1000,"used":500}]}`)+ fmt.Fprint(w, `{"data":[{"storage":"local","type":"dir","content":"images,backup","total":1000,"used":500}]}`)}))defer server.Close()diff --git a/pkg/proxmox/cluster_client_api_test.go b/pkg/proxmox/cluster_client_api_test.goindex 4472d2cb1d..b202f68c8b 100644--- a/pkg/proxmox/cluster_client_api_test.go+++ b/pkg/proxmox/cluster_client_api_test.go@@ -449,7 +449,7 @@ func TestClusterClient_GetStorage(t *testing.T) {fmt.Fprint(w, `{"data":[{"node":"node1","status":"online"}]}`)return}- fmt.Fprint(w, `{"data":[{"storage":"local","type":"dir"}]}`)+ fmt.Fprint(w, `{"data":[{"storage":"local","type":"dir","content":"images,backup"}]}`)}))defer server.Close()diff --git a/pkg/proxmox/storage_inventory_response.go b/pkg/proxmox/storage_inventory_response.gonew file mode 100644index 0000000000..3486e7f6d8--- /dev/null+++ b/pkg/proxmox/storage_inventory_response.go@@ -0,0 +1,83 @@+package proxmox++import (+ "bytes"+ "encoding/json"+ "errors"+ "net/http"+ "strings"+ "unicode/utf8"+)++// These inventories govern storage and backup removal. A malformed success is+// not an empty enumeration or a permission refusal; never echo its body.+var errInvalidStorageInventoryResponse = errors.New("invalid Proxmox storage inventory response")++func readStorageCatalog(resp *http.Response, nodeScoped bool) ([]Storage, error) {+ return readStorageInventory(resp, []string{+ "storage", "type", "content", "active", "enabled", "shared", "nodes", "pool", "path", "total", "used", "avail",+ }, func(storage Storage) (string, bool) {+ // /storage is configuration: optional defaults need not be expanded.+ // /nodes/{node}/storage declares type and supported content; without+ // those discriminators a backup target could silently be skipped.+ valid := strings.TrimSpace(storage.Storage) != ""+ if nodeScoped {+ valid = valid && strings.TrimSpace(storage.Type) != "" && strings.TrimSpace(storage.Content) != ""+ }+ return storage.Storage, valid+ })+}++func readStorageContentInventory(resp *http.Response) ([]StorageContent, error) {+ return readStorageInventory(resp, []string{+ "volid", "content", "ctime", "format", "size", "used", "vmid", "notes", "protected", "encryption", "verification", "verified",+ }, func(content StorageContent) (string, bool) {+ // Validate every row before the caller filters content kinds. A missing+ // discriminator is unread history, not an uninteresting file type.+ return content.Volid, strings.TrimSpace(content.Volid) != "" && strings.TrimSpace(content.Content) != ""+ })+}++func readStorageInventory[T any](resp *http.Response, fields []string, identity func(T) (string, bool)) ([]T, error) {+ body, readErr := readResponseBodyLimited(resp.Body)+ closeErr := resp.Body.Close()+ if readErr != nil || closeErr != nil {+ return nil, errors.Join(readErr, closeErr)+ }+ if resp.StatusCode != http.StatusOK || !utf8.Valid(body) {+ return nil, errInvalidStorageInventoryResponse+ }+ envelope, valid := inventoryResponseObject(body, "data", "errors")+ if !valid {+ return nil, errInvalidStorageInventoryResponse+ }+ if failure, exists := envelope["errors"]; exists && !bytes.Equal(bytes.TrimSpace(failure), []byte("null")) {+ return nil, errInvalidStorageInventoryResponse+ }+ data := bytes.TrimSpace(envelope["data"])+ if len(data) == 0 || data[0] != '[' {+ return nil, errInvalidStorageInventoryResponse+ }+ var rows []json.RawMessage+ if json.Unmarshal(data, &rows) != nil {+ return nil, errInvalidStorageInventoryResponse+ }+ values := make([]T, 0, len(rows))+ seen := make(map[string]bool, len(rows))+ for _, row := range rows {+ if _, valid := inventoryResponseObject(row, fields...); !valid {+ return nil, errInvalidStorageInventoryResponse+ }+ var value T+ if json.Unmarshal(row, &value) != nil {+ return nil, errInvalidStorageInventoryResponse+ }+ key, valid := identity(value)+ if !valid || seen[key] {+ return nil, errInvalidStorageInventoryResponse+ }+ seen[key] = true+ values = append(values, value)+ }+ return values, nil+}diff --git a/pkg/proxmox/storage_inventory_response_test.go b/pkg/proxmox/storage_inventory_response_test.gonew file mode 100644index 0000000000..ea100d9b57--- /dev/null+++ b/pkg/proxmox/storage_inventory_response_test.go@@ -0,0 +1,187 @@+package proxmox++import (+ "context"+ "fmt"+ "io"+ "net/http"+ "net/http/httptest"+ "strings"+ "sync/atomic"+ "testing"+)++func readTestStorageInventory(c *Client, kind string) (int, bool, error) {+ if kind == "content" {+ rows, err := c.GetStorageContent(context.Background(), "member", "archive")+ return len(rows), rows == nil, err+ }+ var rows []Storage+ var err error+ if kind == "node" {+ rows, err = c.GetStorage(context.Background(), "member")+ } else {+ rows, err = c.GetAllStorage(context.Background())+ }+ return len(rows), rows == nil, err+}++func TestStorageInventoryResponseAdmission(t *testing.T) {+ for _, kind := range []string{"node", "configuration", "content"} {+ t.Run(kind, func(t *testing.T) {+ row, id, endpoint := `{"storage":"archive","type":"dir","content":"backup"}`, "storage", "/api2/json/storage"+ if kind == "node" {+ endpoint = "/api2/json/nodes/member/storage"+ } else if kind == "content" {+ row, id, endpoint = `{"volid":"archive:backup/accepted","content":"backup","ctime":1700000000}`, "volid", "/api2/json/nodes/member/storage/archive/content"+ }+ inventory := `{"data":[` + row + `]}`+ cases := []struct {+ name, body string+ incomplete bool+ }{+ {"missing-data", `{}`, false},+ {"null-data", `{"data":null}`, false},+ {"null-envelope", `null`, false},+ {"non-array", `{"data":{}}`, false},+ {"error-as-success", `{"errors":{"message":"fixture-private-detail"}}`, false},+ {"conflicting-error", `{"data":[],"errors":{"message":"fixture-private-detail"}}`, false},+ {"case-data", `{"Data":[]}`, false},+ {"duplicate-data", `{"data":[` + row + `],"data":[]}`, false},+ {"escaped-duplicate-data", `{"data":[],"\u0064ata":[` + row + `]}`, false},+ {"case-overwrite", `{"data":[` + row + `],"Data":[]}`, false},+ {"case-error", `{"data":[],"Errors":null}`, false},+ {"trailing-value", inventory + ` {"data":[]}`, false},+ {"trailing-garbage", inventory + `fixture-private-detail`, false},+ {"truncated-after-prefix", inventory, true},+ {"oversized-after-prefix", inventory + strings.Repeat(" ", int(maxResponseBodyBytes)), false},+ {"invalid-utf8", `{"data":[` + row + `],"metadata":"` + string([]byte{0xff}) + `"}`, false},+ {"null-row", `{"data":[` + row + `,null]}`, false},+ {"missing-identity", `{"data":[{"content":"backup","type":"dir"}]}`, false},+ {"empty-identity", `{"data":[{"` + id + `":"","content":"backup","type":"dir"}]}`, false},+ {"null-identity", `{"data":[{"` + id + `":null,"content":"backup","type":"dir"}]}`, false},+ {"whitespace-identity", `{"data":[{"` + id + `":" ","content":"backup","type":"dir"}]}`, false},+ {"duplicate-identity-field", `{"data":[{"` + id + `":"other",` + row[1:] + `]}`, false},+ {"case-identity-field", `{"data":[{"` + strings.ToUpper(id) + `":"other",` + row[1:] + `]}`, false},+ {"case-content-field", `{"data":[{"Content":"iso",` + row[1:] + `]}`, false},+ {"duplicate-row", `{"data":[` + row + `,` + row + `]}`, false},+ {"bad-row-after-good", `{"data":[` + row + `,{"` + id + `":"second","content":false}]}`, false},+ }+ if kind != "configuration" {+ for _, value := range []string{"", `,"content":null`, `,"content":""`, `,"content":" "`} {+ cases = append(cases, struct {+ name, body string+ incomplete bool+ }{"unread-content/" + value, `{"data":[{"` + id + `":"accepted","type":"dir"` + value + `}]}`, false})+ }+ }+ if kind == "node" {+ cases = append(cases, struct {+ name, body string+ incomplete bool+ }{"missing-type", `{"data":[{"storage":"archive","content":"backup"}]}`, false})+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ var requests atomic.Int32+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ requests.Add(1)+ if r.URL.Path != endpoint {+ t.Errorf("unexpected path: %s", r.URL.Path)+ }+ if tc.incomplete {+ w.Header().Set("Content-Length", fmt.Sprint(len(tc.body)+100))+ } else {+ w.(http.Flusher).Flush()+ }+ fmt.Fprint(w, tc.body)+ }))+ defer server.Close()+ count, nilRows, err := readTestStorageInventory(backupTestClient(t, server.URL), kind)+ if err == nil || !nilRows || count != 0 {+ t.Errorf("unverified inventory admitted: count=%d nil=%t err=%v", count, nilRows, err)+ }+ if err != nil && strings.Contains(err.Error(), "fixture-private-detail") {+ t.Error("response body leaked into storage diagnostic")+ }+ if requests.Load() != 1 {+ t.Errorf("invalid success replayed: %d requests", requests.Load())+ }+ })+ }+ })+ }+}++func TestStorageInventoryAcceptedScopedAndExtendedResponses(t *testing.T) {+ for _, kind := range []string{"node", "configuration", "content"} {+ for _, body := range []string{`{"data":[]}`, `{"data":[],"errors":null,"total":0}`} {+ t.Run(kind+"/"+body, func(t *testing.T) {+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, body) }))+ defer server.Close()+ count, _, err := readTestStorageInventory(backupTestClient(t, server.URL), kind)+ if err != nil || count != 0 {+ t.Fatalf("scoped empty inventory refused: count=%d err=%v", count, err)+ }+ })+ }+ }+ t.Run("node-fields", func(t *testing.T) {+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ fmt.Fprint(w, `{"data":[{"storage":"archive","type":"dir","content":"backup,images","active":1,"enabled":1,"shared":1,"nodes":"member,peer","pool":"tank","path":"/archive","total":100,"used":0,"avail":100,"provider-metadata":{"revision":1}}],"errors":null}`)+ }))+ defer server.Close()+ rows, err := backupTestClient(t, server.URL).GetStorage(context.Background(), "member")+ if err != nil || len(rows) != 1 || rows[0] != (Storage{Storage: "archive", Type: "dir", Content: "backup,images", Active: 1, Enabled: 1, Shared: 1, Nodes: "member,peer", Pool: "tank", Path: "/archive", Total: 100, Available: 100}) {+ t.Fatalf("accepted storage fields changed: %+v err=%v", rows, err)+ }+ })+ t.Run("configuration-defaults", func(t *testing.T) {+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, `{"data":[{"storage":"archive"}]}`) }))+ defer server.Close()+ count, _, err := readTestStorageInventory(backupTestClient(t, server.URL), "configuration")+ if err != nil || count != 1 {+ t.Fatalf("configuration defaults refused: count=%d err=%v", count, err)+ }+ })+ t.Run("all-content-validated-before-filter", func(t *testing.T) {+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ fmt.Fprint(w, `{"data":[{"volid":"archive:backup/accepted","content":"backup","ctime":1700000000,"size":0,"vmid":101,"format":"pbs-vm","protected":1,"verified":1,"verification":{"state":"ok"},"notes":"accepted"},{"volid":"archive:backup/host","content":"backup","vmid":0},{"volid":"archive:vztmpl/template","content":"vztmpl"},{"volid":"archive:iso/image","content":"iso"},{"volid":"archive:future/item","content":"future-kind","future-metadata":1}]}`)+ }))+ defer server.Close()+ rows, err := backupTestClient(t, server.URL).GetStorageContent(context.Background(), "member", "archive")+ if err != nil || len(rows) != 3 || rows[0].CTime != 1700000000 || rows[0].VMID != 101 || rows[0].Size != 0 || rows[0].Format != "pbs-vm" || rows[0].Protected != 1 || rows[0].Verified != 1 || rows[0].Verification["state"] != "ok" || rows[0].Notes != "accepted" || rows[1].VMID != 0 || rows[2].Content != "vztmpl" {+ t.Fatalf("accepted backup/template filter or fields changed: %+v err=%v", rows, err)+ }+ })+}++func TestStorageInventoryRequiresCompletedResponse(t *testing.T) {+ for _, status := range []int{http.StatusNoContent, http.StatusPartialContent} {+ for _, content := range []bool{false, true} {+ resp := &http.Response{StatusCode: status, Body: io.NopCloser(strings.NewReader(`{"data":[]}`))}+ var err error+ if content {+ _, err = readStorageContentInventory(resp)+ } else {+ _, err = readStorageCatalog(resp, true)+ }+ if err == nil {+ t.Errorf("status %d established absence (content=%t)", status, content)+ }+ }+ }+ for _, content := range []bool{false, true} {+ body := &snapshotCloseFailure{Reader: strings.NewReader(`{"data":[]}`)}+ resp := &http.Response{StatusCode: http.StatusOK, Body: body}+ var err error+ if content {+ _, err = readStorageContentInventory(resp)+ } else {+ _, err = readStorageCatalog(resp, true)+ }+ if err == nil || !body.closed {+ t.Errorf("unfinished close established absence (content=%t): err=%v closed=%t", content, err, body.closed)+ }+ }+}