Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Reject incomplete Proxmox storage inventories

pulse-triage[bot]ce1fc7e5ff9c
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index 7d3aa81132..62197dd277 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -8389,3 +8389,26 @@ complete recovery and confirmed scoped removal. `TestPVESnapshotHTTPBrowserProdu
feeds the actual consuming proof. These are synthetic HTTP/provider fixtures,
not native guest safety, backup execution, installed recovery or a diagnosis of
#3248. The separate prior lifecycle-suite failure remains unexplained.
+
+### Storage HTTP success must be a complete inventory
+
+PVE node storage, cluster storage configuration and storage content readers admit
+only a fully read, bounded HTTP-200 response with one unambiguous JSON object and
+an explicit `data` array. Missing/null inventories, error-bearing or conflicting
+envelopes, trailing bytes, invalid rows and duplicate identities are unread
+history, not successful empty or partial enumerations. Every content row is
+validated before filtering backup/template kinds. Node catalogs require their
+storage identity, type and supported-content discriminators; configuration-only
+`/storage` rows retain optional/default fields. Optional metadata, host backups,
+zero-byte/in-progress entries and explicit empty arrays remain supported. No
+schema, transport, auth, failover, query admission or polling budget changes.
+
+`TestStorageInventoryResponseAdmission`, accepted-field/filter controls and
+completed-response tests exercise all three HTTP readers. The registered
+`TestPVEStorageHTTP*` regressions connect real HTTP reads to storage backup
+polling, asynchronous ingest and canonical protection: warm/cold history,
+independently accepted peers, all-unread state, repeated failures, ordinary
+recovery and confirmed scoped absence. Their opt-in producer feeds the existing
+production Backups consumer for browser proof. Retained evidence is not a fresh
+read, available archive, installed recovery or a diagnosis of #3248; no guest or
+backup operation, thaw or filesystem-liveness assertion follows.
diff --git a/internal/monitoring/storage_inventory_http_test.go b/internal/monitoring/storage_inventory_http_test.go
new file mode 100644
index 0000000000..5034d12b69
--- /dev/null
+++ b/internal/monitoring/storage_inventory_http_test.go
@@ -0,0 +1,299 @@
+package monitoring
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "reflect"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/models"
+ "github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust"
+ "github.com/rcourtman/pulse-go-rewrite/internal/recovery"
+ "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
+)
+
+// Only storage catalog/content reads go through the real HTTP client. No guest
+// commands, backup operation, external service or database-internal change.
+type storageInventoryHTTPClient struct {
+ *pveReconciliationClient
+ api *proxmox.Client
+}
+
+func (c *storageInventoryHTTPClient) GetStorage(ctx context.Context, node string) ([]proxmox.Storage, error) {
+ return c.api.GetStorage(ctx, node)
+}
+
+func (c *storageInventoryHTTPClient) GetStorageContent(ctx context.Context, node, storage string) ([]proxmox.StorageContent, error) {
+ return c.api.GetStorageContent(ctx, node, storage)
+}
+
+type storageInventoryHTTPFixture struct {
+ f *pveReconciliationFixture
+ client *storageInventoryHTTPClient
+ mu sync.Mutex
+ bodies map[string]string
+ paths []string
+}
+
+func storageInventoryHTTPPath(kind, node string) string {
+ path := "/api2/json/nodes/" + node + "/storage"
+ if kind == "content" {
+ path += "/archive-" + node + "/content"
+ }
+ return path
+}
+
+func newStorageInventoryHTTPFixture(t *testing.T) *storageInventoryHTTPFixture {
+ t.Helper()
+ h := &storageInventoryHTTPFixture{f: newPVEReconciliationFixture(t), bodies: make(map[string]string)}
+ // Normal storage inventory supplies the carry-forward lookup when a node's
+ // catalog cannot be read. Cold-state cases deliberately discard that cache.
+ h.f.m.state.UpdateStorage([]models.Storage{
+ {ID: "cluster-node-a-archive-node-a", Name: "archive-node-a", Node: "node-a", Instance: "cluster", Content: "backup", Type: "dir"},
+ {ID: "cluster-node-b-archive-node-b", Name: "archive-node-b", Node: "node-b", Instance: "cluster", Content: "backup", Type: "dir"},
+ })
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ h.mu.Lock()
+ defer h.mu.Unlock()
+ h.paths = append(h.paths, r.URL.Path)
+ w.Header().Set("Content-Type", "application/json")
+ if body, ok := h.bodies[r.URL.Path]; ok {
+ fmt.Fprint(w, body)
+ return
+ }
+ for _, node := range []string{"node-a", "node-b"} {
+ var data any
+ if r.URL.Path == storageInventoryHTTPPath("catalog", node) {
+ data, _ = h.f.client.GetStorage(r.Context(), node)
+ } else if r.URL.Path == storageInventoryHTTPPath("content", node) {
+ data, _ = h.f.client.GetStorageContent(r.Context(), node, "archive-"+node)
+ } else {
+ continue
+ }
+ if err := json.NewEncoder(w).Encode(map[string]any{"data": data}); err != nil {
+ t.Error(err)
+ }
+ return
+ }
+ t.Errorf("unexpected fixture request: %s", r.URL.Path)
+ http.NotFound(w, r)
+ }))
+ t.Cleanup(server.Close)
+ api, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: 5 * time.Second})
+ if err != nil {
+ t.Fatal(err)
+ }
+ h.client = &storageInventoryHTTPClient{pveReconciliationClient: h.f.client, api: api}
+ return h
+}
+
+func (h *storageInventoryHTTPFixture) set(kind, node, body string) {
+ h.mu.Lock()
+ defer h.mu.Unlock()
+ if body == "" {
+ delete(h.bodies, storageInventoryHTTPPath(kind, node))
+ } else {
+ h.bodies[storageInventoryHTTPPath(kind, node)] = body
+ }
+}
+
+func (h *storageInventoryHTTPFixture) advance() {
+ h.mu.Lock()
+ defer h.mu.Unlock()
+ h.f.client.at = h.f.client.at.Add(15 * time.Minute)
+}
+
+func (h *storageInventoryHTTPFixture) poll(t *testing.T, wantReads int) {
+ t.Helper()
+ h.mu.Lock()
+ before := len(h.paths)
+ h.mu.Unlock()
+ h.f.m.pollStorageBackupsWithNodes(context.Background(), "cluster", h.client, h.f.nodes, h.f.online)
+ h.f.wait(t)
+ h.mu.Lock()
+ defer h.mu.Unlock()
+ if len(h.paths)-before != wantReads {
+ t.Fatalf("got HTTP reads %v, want %d; no retries or other operations", h.paths[before:], wantReads)
+ }
+}
+
+func TestPVEStorageHTTPInvalidResponseRetainsHistory(t *testing.T) {
+ for _, kind := range []string{"catalog", "content"} {
+ row := `{"storage":"archive-node-b","type":"dir","content":"backup","enabled":1,"active":1}`
+ missingDiscriminator := `{"storage":"archive-node-b","type":"dir"}`
+ if kind == "content" {
+ row = `{"volid":"archive-node-b:backup/vzdump-qemu-102.vma.zst","vmid":102,"content":"backup","ctime":1700000000}`
+ missingDiscriminator = `{"volid":"archive-node-b:backup/vzdump-qemu-102.vma.zst"}`
+ }
+ cases := map[string]string{
+ "missing-data": `{}`,
+ "null-data": `{"data":null}`,
+ "conflicting-error": `{"data":[],"errors":{"message":"fixture-private-detail"}}`,
+ "duplicate-data": `{"data":[` + row + `],"data":[]}`,
+ "trailing-value": `{"data":[]} {"data":[]}`,
+ "null-row": `{"data":[null]}`,
+ "bad-row-after-good": `{"data":[` + row + `,null]}`,
+ "missing-discriminator": `{"data":[` + missingDiscriminator + `]}`,
+ }
+ for _, cold := range []bool{false, true} {
+ for name, body := range cases {
+ t.Run(fmt.Sprintf("%s/cold=%t/%s", kind, cold, name), func(t *testing.T) {
+ h := newStorageInventoryHTTPFixture(t)
+ f := h.f
+ h.poll(t, 4)
+ other := recoveryIngestTestPoint("pve-backup:unrelated", "other-site", f.client.at)
+ if err := f.store.UpsertPoints(context.Background(), []recovery.RecoveryPoint{other}); err != nil {
+ t.Fatal(err)
+ }
+ before := f.points(t)
+ if len(before) != 3 {
+ t.Fatalf("healthy control points=%d, want 3", len(before))
+ }
+ beforeRows := f.m.state.GetSnapshot().PVEBackups.StorageBackups
+ if cold {
+ f.coldState()
+ }
+ h.advance()
+ h.set(kind, "node-b", body)
+ reads := 4
+ if kind == "catalog" {
+ reads = 3
+ }
+ for range 2 {
+ h.poll(t, reads)
+ after := f.points(t)
+ if len(after) != 3 || !reflect.DeepEqual(before[other.ID], after[other.ID]) {
+ t.Fatal("malformed success deleted history, admitted extra rows or changed another instance")
+ }
+ for id, point := range before {
+ if point.SubjectResourceID == f.ids[1] && !reflect.DeepEqual(point, after[id]) {
+ t.Fatal("unreadable backup was removed or its observation renewed")
+ }
+ if point.SubjectResourceID == f.ids[0] && (after[id].CompletedAt == nil || !after[id].CompletedAt.Equal(f.client.at)) {
+ t.Fatal("independent accepted storage did not update")
+ }
+ }
+ f.assertHistory(t, recovery.ProtectionHistoryPartial, operationaltrust.EvidencePermissionsUnknown)
+ rows := f.m.state.GetSnapshot().PVEBackups.StorageBackups
+ want := 2
+ if cold {
+ want = 1
+ }
+ if len(rows) != want {
+ t.Fatalf("live rows=%d want=%d; unread history cannot erase warm rows or invent cold ones", len(rows), want)
+ }
+ if !cold {
+ for _, row := range rows {
+ if row.VMID == 102 && !reflect.DeepEqual(row, beforeRows[1]) {
+ t.Fatal("unread HTTP inventory changed its warm row")
+ }
+ }
+ }
+ }
+ h.set(kind, "node-b", "")
+ h.poll(t, 4)
+ f.assertHistory(t, recovery.ProtectionHistoryComplete, operationaltrust.EvidencePermissionsSufficient)
+ if len(f.m.state.GetSnapshot().PVEBackups.StorageBackups) != 2 {
+ t.Fatal("accepted recovery did not restore live inventory")
+ }
+ // Explicit scoped absence still clears only this connection.
+ h.set(kind, "node-a", `{"data":[]}`)
+ h.set(kind, "node-b", `{"data":[]}`)
+ reads = 4
+ if kind == "catalog" {
+ reads = 2
+ }
+ h.poll(t, reads)
+ if points := f.points(t); len(points) != 1 || !reflect.DeepEqual(before[other.ID], points[other.ID]) || len(f.m.state.GetSnapshot().PVEBackups.StorageBackups) != 0 {
+ t.Fatal("explicit empty response failed to clear exactly its own backup scope")
+ }
+ })
+ }
+ }
+ }
+}
+
+func TestPVEStorageHTTPAllUnreadCannotClaimEmpty(t *testing.T) {
+ for _, kind := range []string{"catalog", "content"} {
+ for _, cold := range []bool{false, true} {
+ t.Run(fmt.Sprintf("%s/cold=%t", kind, cold), func(t *testing.T) {
+ h := newStorageInventoryHTTPFixture(t)
+ h.poll(t, 4)
+ before := h.f.points(t)
+ if cold {
+ h.f.coldState()
+ }
+ for _, node := range []string{"node-a", "node-b"} {
+ h.set(kind, node, `{"data":null}`)
+ }
+ reads := 4
+ if kind == "catalog" {
+ reads = 2
+ }
+ for range 2 {
+ h.poll(t, reads)
+ if !reflect.DeepEqual(before, h.f.points(t)) {
+ t.Fatal("all-unread response removed or renewed recovery evidence")
+ }
+ h.f.assertHistory(t, recovery.ProtectionHistoryUnavailable, operationaltrust.EvidencePermissionsUnknown)
+ }
+ })
+ }
+ }
+}
+
+// The browser consumes these actual HTTP -> poll -> queue -> store -> posture
+// frames, not a separately invented provider state. Export stays opt-in.
+func TestPVEStorageHTTPBrowserProducer(t *testing.T) {
+ if os.Getenv("PULSE_STORAGE_HTTP_FRAMES") != "1" {
+ t.Skip("producer export is opt-in")
+ }
+ h := newStorageInventoryHTTPFixture(t)
+ frames := make(map[string]any)
+ for _, phase := range []string{"observed", "catalog-unread", "content-unread", "cold-unread", "recovered", "confirmed-empty"} {
+ switch phase {
+ case "catalog-unread":
+ h.set("catalog", "node-b", `{}`)
+ h.advance()
+ case "content-unread":
+ h.set("catalog", "node-b", "")
+ h.set("content", "node-b", `{"data":[{"volid":"unaccepted","content":"backup"},null]}`)
+ case "cold-unread":
+ h.f.coldState()
+ case "recovered":
+ h.set("content", "node-b", "")
+ case "confirmed-empty":
+ h.set("content", "node-a", `{"data":[]}`)
+ h.set("content", "node-b", `{"data":[]}`)
+ }
+ reads := 4
+ if phase == "catalog-unread" {
+ reads = 3
+ }
+ h.poll(t, reads)
+ points := h.f.points(t)
+ want := 2
+ if phase == "confirmed-empty" {
+ want = 0
+ }
+ if len(points) != want {
+ t.Fatalf("phase %s points=%d want=%d", phase, len(points), want)
+ }
+ frames[phase] = map[string]any{"pve": map[string]any{"data": h.f.m.state.GetSnapshot().PVEBackups},
+ "postures": map[string]any{"data": h.f.postures(t), "policy": recovery.DefaultProtectionPosturePolicy.Payload()}, "points": points}
+ }
+ data, err := json.Marshal(map[string]any{"observed_at": time.Now().UTC(), "resource_ids": h.f.ids, "frames": frames, "paths": h.paths})
+ if err != nil {
+ t.Fatal(err)
+ }
+ // One Go-JSON event may hold only part of this line; collectors must join
+ // all Output events before extracting the producer frame.
+ t.Log("STORAGE_HTTP_FRAMES=" + strings.TrimSpace(string(data)))
+}
diff --git a/pkg/proxmox/client.go b/pkg/proxmox/client.go
index eb342ba1a7..c3b27c202c 100644
--- a/pkg/proxmox/client.go
+++ b/pkg/proxmox/client.go
@@ -1357,17 +1357,7 @@ func (c *Client) GetStorage(ctx context.Context, node string) ([]Storage, error)
if err != nil {
return nil, err
}
- defer resp.Body.Close()
-
- var result struct {
- Data []Storage `json:"data"`
- }
-
- if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
- return nil, err
- }
-
- return result.Data, nil
+ return readStorageCatalog(resp, true)
}
// GetAllStorage returns storage information across all nodes
@@ -1387,17 +1377,7 @@ func (c *Client) GetAllStorage(ctx context.Context) ([]Storage, error) {
if err != nil {
return nil, err
}
- defer resp.Body.Close()
-
- var result struct {
- Data []Storage `json:"data"`
- }
-
- if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
- return nil, err
- }
-
- return result.Data, nil
+ return readStorageCatalog(resp, false)
}
// Task represents a Proxmox task
@@ -1561,19 +1541,14 @@ func (c *Client) GetStorageContent(ctx context.Context, node, storage string) ([
if err != nil {
return nil, err
}
- defer resp.Body.Close()
-
- var result struct {
- Data []StorageContent `json:"data"`
- }
-
- if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
+ contents, err := readStorageContentInventory(resp)
+ if err != nil {
return nil, err
}
// Filter for backup content only
var backups []StorageContent
- for _, content := range result.Data {
+ for _, content := range contents {
if content.Content == "backup" || content.Content == "vztmpl" {
backups = append(backups, content)
}
diff --git a/pkg/proxmox/client_api_more4_test.go b/pkg/proxmox/client_api_more4_test.go
index f46797b087..0f0fd5cd48 100644
--- a/pkg/proxmox/client_api_more4_test.go
+++ b/pkg/proxmox/client_api_more4_test.go
@@ -132,7 +132,7 @@ func TestClient_GetContainers(t *testing.T) {
func TestClient_GetStorage(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
- fmt.Fprint(w, `{"data":[{"storage":"local","type":"dir","total":1000,"used":500}]}`)
+ fmt.Fprint(w, `{"data":[{"storage":"local","type":"dir","content":"images,backup","total":1000,"used":500}]}`)
}))
defer server.Close()
diff --git a/pkg/proxmox/cluster_client_api_test.go b/pkg/proxmox/cluster_client_api_test.go
index 4472d2cb1d..b202f68c8b 100644
--- a/pkg/proxmox/cluster_client_api_test.go
+++ b/pkg/proxmox/cluster_client_api_test.go
@@ -449,7 +449,7 @@ func TestClusterClient_GetStorage(t *testing.T) {
fmt.Fprint(w, `{"data":[{"node":"node1","status":"online"}]}`)
return
}
- fmt.Fprint(w, `{"data":[{"storage":"local","type":"dir"}]}`)
+ fmt.Fprint(w, `{"data":[{"storage":"local","type":"dir","content":"images,backup"}]}`)
}))
defer server.Close()
diff --git a/pkg/proxmox/storage_inventory_response.go b/pkg/proxmox/storage_inventory_response.go
new file mode 100644
index 0000000000..3486e7f6d8
--- /dev/null
+++ b/pkg/proxmox/storage_inventory_response.go
@@ -0,0 +1,83 @@
+package proxmox
+
+import (
+ "bytes"
+ "encoding/json"
+ "errors"
+ "net/http"
+ "strings"
+ "unicode/utf8"
+)
+
+// These inventories govern storage and backup removal. A malformed success is
+// not an empty enumeration or a permission refusal; never echo its body.
+var errInvalidStorageInventoryResponse = errors.New("invalid Proxmox storage inventory response")
+
+func readStorageCatalog(resp *http.Response, nodeScoped bool) ([]Storage, error) {
+ return readStorageInventory(resp, []string{
+ "storage", "type", "content", "active", "enabled", "shared", "nodes", "pool", "path", "total", "used", "avail",
+ }, func(storage Storage) (string, bool) {
+ // /storage is configuration: optional defaults need not be expanded.
+ // /nodes/{node}/storage declares type and supported content; without
+ // those discriminators a backup target could silently be skipped.
+ valid := strings.TrimSpace(storage.Storage) != ""
+ if nodeScoped {
+ valid = valid && strings.TrimSpace(storage.Type) != "" && strings.TrimSpace(storage.Content) != ""
+ }
+ return storage.Storage, valid
+ })
+}
+
+func readStorageContentInventory(resp *http.Response) ([]StorageContent, error) {
+ return readStorageInventory(resp, []string{
+ "volid", "content", "ctime", "format", "size", "used", "vmid", "notes", "protected", "encryption", "verification", "verified",
+ }, func(content StorageContent) (string, bool) {
+ // Validate every row before the caller filters content kinds. A missing
+ // discriminator is unread history, not an uninteresting file type.
+ return content.Volid, strings.TrimSpace(content.Volid) != "" && strings.TrimSpace(content.Content) != ""
+ })
+}
+
+func readStorageInventory[T any](resp *http.Response, fields []string, identity func(T) (string, bool)) ([]T, error) {
+ body, readErr := readResponseBodyLimited(resp.Body)
+ closeErr := resp.Body.Close()
+ if readErr != nil || closeErr != nil {
+ return nil, errors.Join(readErr, closeErr)
+ }
+ if resp.StatusCode != http.StatusOK || !utf8.Valid(body) {
+ return nil, errInvalidStorageInventoryResponse
+ }
+ envelope, valid := inventoryResponseObject(body, "data", "errors")
+ if !valid {
+ return nil, errInvalidStorageInventoryResponse
+ }
+ if failure, exists := envelope["errors"]; exists && !bytes.Equal(bytes.TrimSpace(failure), []byte("null")) {
+ return nil, errInvalidStorageInventoryResponse
+ }
+ data := bytes.TrimSpace(envelope["data"])
+ if len(data) == 0 || data[0] != '[' {
+ return nil, errInvalidStorageInventoryResponse
+ }
+ var rows []json.RawMessage
+ if json.Unmarshal(data, &rows) != nil {
+ return nil, errInvalidStorageInventoryResponse
+ }
+ values := make([]T, 0, len(rows))
+ seen := make(map[string]bool, len(rows))
+ for _, row := range rows {
+ if _, valid := inventoryResponseObject(row, fields...); !valid {
+ return nil, errInvalidStorageInventoryResponse
+ }
+ var value T
+ if json.Unmarshal(row, &value) != nil {
+ return nil, errInvalidStorageInventoryResponse
+ }
+ key, valid := identity(value)
+ if !valid || seen[key] {
+ return nil, errInvalidStorageInventoryResponse
+ }
+ seen[key] = true
+ values = append(values, value)
+ }
+ return values, nil
+}
diff --git a/pkg/proxmox/storage_inventory_response_test.go b/pkg/proxmox/storage_inventory_response_test.go
new file mode 100644
index 0000000000..ea100d9b57
--- /dev/null
+++ b/pkg/proxmox/storage_inventory_response_test.go
@@ -0,0 +1,187 @@
+package proxmox
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "sync/atomic"
+ "testing"
+)
+
+func readTestStorageInventory(c *Client, kind string) (int, bool, error) {
+ if kind == "content" {
+ rows, err := c.GetStorageContent(context.Background(), "member", "archive")
+ return len(rows), rows == nil, err
+ }
+ var rows []Storage
+ var err error
+ if kind == "node" {
+ rows, err = c.GetStorage(context.Background(), "member")
+ } else {
+ rows, err = c.GetAllStorage(context.Background())
+ }
+ return len(rows), rows == nil, err
+}
+
+func TestStorageInventoryResponseAdmission(t *testing.T) {
+ for _, kind := range []string{"node", "configuration", "content"} {
+ t.Run(kind, func(t *testing.T) {
+ row, id, endpoint := `{"storage":"archive","type":"dir","content":"backup"}`, "storage", "/api2/json/storage"
+ if kind == "node" {
+ endpoint = "/api2/json/nodes/member/storage"
+ } else if kind == "content" {
+ row, id, endpoint = `{"volid":"archive:backup/accepted","content":"backup","ctime":1700000000}`, "volid", "/api2/json/nodes/member/storage/archive/content"
+ }
+ inventory := `{"data":[` + row + `]}`
+ cases := []struct {
+ name, body string
+ incomplete bool
+ }{
+ {"missing-data", `{}`, false},
+ {"null-data", `{"data":null}`, false},
+ {"null-envelope", `null`, false},
+ {"non-array", `{"data":{}}`, false},
+ {"error-as-success", `{"errors":{"message":"fixture-private-detail"}}`, false},
+ {"conflicting-error", `{"data":[],"errors":{"message":"fixture-private-detail"}}`, false},
+ {"case-data", `{"Data":[]}`, false},
+ {"duplicate-data", `{"data":[` + row + `],"data":[]}`, false},
+ {"escaped-duplicate-data", `{"data":[],"\u0064ata":[` + row + `]}`, false},
+ {"case-overwrite", `{"data":[` + row + `],"Data":[]}`, false},
+ {"case-error", `{"data":[],"Errors":null}`, false},
+ {"trailing-value", inventory + ` {"data":[]}`, false},
+ {"trailing-garbage", inventory + `fixture-private-detail`, false},
+ {"truncated-after-prefix", inventory, true},
+ {"oversized-after-prefix", inventory + strings.Repeat(" ", int(maxResponseBodyBytes)), false},
+ {"invalid-utf8", `{"data":[` + row + `],"metadata":"` + string([]byte{0xff}) + `"}`, false},
+ {"null-row", `{"data":[` + row + `,null]}`, false},
+ {"missing-identity", `{"data":[{"content":"backup","type":"dir"}]}`, false},
+ {"empty-identity", `{"data":[{"` + id + `":"","content":"backup","type":"dir"}]}`, false},
+ {"null-identity", `{"data":[{"` + id + `":null,"content":"backup","type":"dir"}]}`, false},
+ {"whitespace-identity", `{"data":[{"` + id + `":" ","content":"backup","type":"dir"}]}`, false},
+ {"duplicate-identity-field", `{"data":[{"` + id + `":"other",` + row[1:] + `]}`, false},
+ {"case-identity-field", `{"data":[{"` + strings.ToUpper(id) + `":"other",` + row[1:] + `]}`, false},
+ {"case-content-field", `{"data":[{"Content":"iso",` + row[1:] + `]}`, false},
+ {"duplicate-row", `{"data":[` + row + `,` + row + `]}`, false},
+ {"bad-row-after-good", `{"data":[` + row + `,{"` + id + `":"second","content":false}]}`, false},
+ }
+ if kind != "configuration" {
+ for _, value := range []string{"", `,"content":null`, `,"content":""`, `,"content":" "`} {
+ cases = append(cases, struct {
+ name, body string
+ incomplete bool
+ }{"unread-content/" + value, `{"data":[{"` + id + `":"accepted","type":"dir"` + value + `}]}`, false})
+ }
+ }
+ if kind == "node" {
+ cases = append(cases, struct {
+ name, body string
+ incomplete bool
+ }{"missing-type", `{"data":[{"storage":"archive","content":"backup"}]}`, false})
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ var requests atomic.Int32
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ requests.Add(1)
+ if r.URL.Path != endpoint {
+ t.Errorf("unexpected path: %s", r.URL.Path)
+ }
+ if tc.incomplete {
+ w.Header().Set("Content-Length", fmt.Sprint(len(tc.body)+100))
+ } else {
+ w.(http.Flusher).Flush()
+ }
+ fmt.Fprint(w, tc.body)
+ }))
+ defer server.Close()
+ count, nilRows, err := readTestStorageInventory(backupTestClient(t, server.URL), kind)
+ if err == nil || !nilRows || count != 0 {
+ t.Errorf("unverified inventory admitted: count=%d nil=%t err=%v", count, nilRows, err)
+ }
+ if err != nil && strings.Contains(err.Error(), "fixture-private-detail") {
+ t.Error("response body leaked into storage diagnostic")
+ }
+ if requests.Load() != 1 {
+ t.Errorf("invalid success replayed: %d requests", requests.Load())
+ }
+ })
+ }
+ })
+ }
+}
+
+func TestStorageInventoryAcceptedScopedAndExtendedResponses(t *testing.T) {
+ for _, kind := range []string{"node", "configuration", "content"} {
+ for _, body := range []string{`{"data":[]}`, `{"data":[],"errors":null,"total":0}`} {
+ t.Run(kind+"/"+body, func(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, body) }))
+ defer server.Close()
+ count, _, err := readTestStorageInventory(backupTestClient(t, server.URL), kind)
+ if err != nil || count != 0 {
+ t.Fatalf("scoped empty inventory refused: count=%d err=%v", count, err)
+ }
+ })
+ }
+ }
+ t.Run("node-fields", func(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ fmt.Fprint(w, `{"data":[{"storage":"archive","type":"dir","content":"backup,images","active":1,"enabled":1,"shared":1,"nodes":"member,peer","pool":"tank","path":"/archive","total":100,"used":0,"avail":100,"provider-metadata":{"revision":1}}],"errors":null}`)
+ }))
+ defer server.Close()
+ rows, err := backupTestClient(t, server.URL).GetStorage(context.Background(), "member")
+ if err != nil || len(rows) != 1 || rows[0] != (Storage{Storage: "archive", Type: "dir", Content: "backup,images", Active: 1, Enabled: 1, Shared: 1, Nodes: "member,peer", Pool: "tank", Path: "/archive", Total: 100, Available: 100}) {
+ t.Fatalf("accepted storage fields changed: %+v err=%v", rows, err)
+ }
+ })
+ t.Run("configuration-defaults", func(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, `{"data":[{"storage":"archive"}]}`) }))
+ defer server.Close()
+ count, _, err := readTestStorageInventory(backupTestClient(t, server.URL), "configuration")
+ if err != nil || count != 1 {
+ t.Fatalf("configuration defaults refused: count=%d err=%v", count, err)
+ }
+ })
+ t.Run("all-content-validated-before-filter", func(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ fmt.Fprint(w, `{"data":[{"volid":"archive:backup/accepted","content":"backup","ctime":1700000000,"size":0,"vmid":101,"format":"pbs-vm","protected":1,"verified":1,"verification":{"state":"ok"},"notes":"accepted"},{"volid":"archive:backup/host","content":"backup","vmid":0},{"volid":"archive:vztmpl/template","content":"vztmpl"},{"volid":"archive:iso/image","content":"iso"},{"volid":"archive:future/item","content":"future-kind","future-metadata":1}]}`)
+ }))
+ defer server.Close()
+ rows, err := backupTestClient(t, server.URL).GetStorageContent(context.Background(), "member", "archive")
+ if err != nil || len(rows) != 3 || rows[0].CTime != 1700000000 || rows[0].VMID != 101 || rows[0].Size != 0 || rows[0].Format != "pbs-vm" || rows[0].Protected != 1 || rows[0].Verified != 1 || rows[0].Verification["state"] != "ok" || rows[0].Notes != "accepted" || rows[1].VMID != 0 || rows[2].Content != "vztmpl" {
+ t.Fatalf("accepted backup/template filter or fields changed: %+v err=%v", rows, err)
+ }
+ })
+}
+
+func TestStorageInventoryRequiresCompletedResponse(t *testing.T) {
+ for _, status := range []int{http.StatusNoContent, http.StatusPartialContent} {
+ for _, content := range []bool{false, true} {
+ resp := &http.Response{StatusCode: status, Body: io.NopCloser(strings.NewReader(`{"data":[]}`))}
+ var err error
+ if content {
+ _, err = readStorageContentInventory(resp)
+ } else {
+ _, err = readStorageCatalog(resp, true)
+ }
+ if err == nil {
+ t.Errorf("status %d established absence (content=%t)", status, content)
+ }
+ }
+ }
+ for _, content := range []bool{false, true} {
+ body := &snapshotCloseFailure{Reader: strings.NewReader(`{"data":[]}`)}
+ resp := &http.Response{StatusCode: http.StatusOK, Body: body}
+ var err error
+ if content {
+ _, err = readStorageContentInventory(resp)
+ } else {
+ _, err = readStorageCatalog(resp, true)
+ }
+ if err == nil || !body.closed {
+ t.Errorf("unfinished close established absence (content=%t): err=%v closed=%t", content, err, body.closed)
+ }
+ }
+}