Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Require whole PBS datastore discovery before reconciling backups
pulse-triage[bot]932c17b69a62
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex ea60a08235..f10f72a7b5 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -17,6 +17,30 @@## Purpose+### Datastore discovery cannot turn unread history into absence++PBS datastore enumeration uses the same bounded, fully read and closed HTTP 200+transport and unambiguous envelope admission as namespace, group, snapshot and+writer inventories. Duplicate/case-aliased fields, non-null errors, invalid+UTF-8 and partial/other success statuses are unread, not an accepted empty outer+scope. Validate the entire enumeration before exclusions or any per-datastore+requests. Retain payload-free diagnostics and existing HTML/HTTPS guidance.+Explicit empty arrays and unique future metadata remain valid; filtering,+status collection, permissions, auth, poll budgets and guest operations do not+change.++`TestDatastoreInventoryRequiresWholeResponse` and the complete-body/valid-scope+controls exercise the client, including read/close/bound failures and excluded+stores. Normal-suite `TestPBSDatastoreWholeResponseRetainsEvidence` drives the+public poll and background worker for VM/CT warm/cold-live states: unread+discovery retains raw and durable evidence without renewal or guessed child+reads, withdraws protection without claiming an access denial, and ordinary+complete/empty reads recover and reconcile absence. The consumer lifecycle+exports these exact producer results for desktop/narrow Backups interactions+and pixel inspection. Existing discovery denial, skip, other-connection and+connectivity-fallback controls remain. No original guest recovery, production+occurrence, issue cause or containing release is established by this proof.+### PBS backup listings require the whole responseNamespace, group and snapshot reads accept one unambiguous JSON envelope onlydiff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.mdindex 4efbe0dfa2..8e15447c5f 100644--- a/docs/release-control/v6/internal/subsystems/storage-recovery.md+++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md@@ -21,6 +21,30 @@## Purpose+### Datastore discovery cannot turn unread history into absence++PBS datastore enumeration uses the same bounded, fully read and closed HTTP 200+transport and unambiguous envelope admission as namespace, group, snapshot and+writer inventories. Duplicate/case-aliased fields, non-null errors, invalid+UTF-8 and partial/other success statuses are unread, not an accepted empty outer+scope. Validate the entire enumeration before exclusions or any per-datastore+requests. Retain payload-free diagnostics and existing HTML/HTTPS guidance.+Explicit empty arrays and unique future metadata remain valid; filtering,+status collection, permissions, auth, poll budgets and guest operations do not+change.++`TestDatastoreInventoryRequiresWholeResponse` and the complete-body/valid-scope+controls exercise the client, including read/close/bound failures and excluded+stores. Normal-suite `TestPBSDatastoreWholeResponseRetainsEvidence` drives the+public poll and background worker for VM/CT warm/cold-live states: unread+discovery retains raw and durable evidence without renewal or guessed child+reads, withdraws protection without claiming an access denial, and ordinary+complete/empty reads recover and reconcile absence. The consumer lifecycle+exports these exact producer results for desktop/narrow Backups interactions+and pixel inspection. Existing discovery denial, skip, other-connection and+connectivity-fallback controls remain. No original guest recovery, production+occurrence, issue cause or containing release is established by this proof.+### PBS backup listings require the whole responseNamespace, group and snapshot reads accept one unambiguous JSON envelope onlydiff --git a/internal/monitoring/pbs_datastore_response_test.go b/internal/monitoring/pbs_datastore_response_test.gonew file mode 100644index 0000000000..ceeb5801e1--- /dev/null+++ b/internal/monitoring/pbs_datastore_response_test.go@@ -0,0 +1,156 @@+package monitoring++import (+ "crypto/sha256"+ "encoding/json"+ "fmt"+ "net/http"+ "os"+ "reflect"+ "testing"++ "github.com/rcourtman/pulse-go-rewrite/internal/models"+ "github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust"+ "github.com/rcourtman/pulse-go-rewrite/internal/recovery"+)++func TestPBSDatastoreWholeResponseRetainsEvidence(t *testing.T) {+ for _, kind := range []string{"vm", "ct"} {+ for _, cold := range []bool{false, true} {+ for _, scenario := range []struct {+ name, body string+ status int+ }{+ {name: "error envelope", body: `{"data":[],"errors":{"private":"fixture-only"}}`},+ {name: "discarded datastore", body: `{"data":[{"store":"archive"}],"data":[]}`},+ {name: "overwritten data", body: `{"data":null,"data":[]}`},+ {name: "case data", body: `{"data":null,"Data":[]}`},+ {name: "overwritten errors", body: `{"data":[],"errors":{"private":"fixture-only"},"errors":null}`},+ {name: "invalid utf8", body: "{\"data\":[],\"metadata\":\"\xff\"}"},+ {name: "created", body: `{"data":[]}`, status: http.StatusCreated},+ {name: "partial", body: `{"data":[]}`, status: http.StatusPartialContent},+ } {+ t.Run(fmt.Sprintf("%s/cold=%t/%s", kind, cold, scenario.name), func(t *testing.T) {+ f := newPBSNamespacePollFixture(t, kind)+ f.poll(t, "pbs-main")+ before := f.points(t, "pbs-main")+ if len(before) != 2 {+ t.Fatal("healthy control did not ingest root and tenant")+ }+ if cold {+ snapshot := f.m.state.GetSnapshot()+ f.m.state = models.NewState()+ f.m.state.UpdateVMs(snapshot.VMs)+ f.m.state.UpdateContainers(snapshot.Containers)+ }+ raw := f.m.state.GetSnapshot().PBSBackups+ key := pbsBackupGroupKey{datastore: "archive", namespace: "tenant", backupType: kind, backupID: "101"}+ cacheAt := f.m.pbsBackupCacheTimeFor("pbs-main", key)+ reads := f.backupReads.Load()+ f.datastoreBody.Store(&scenario.body)+ f.datastoreStatus.Store(int32(scenario.status))+ for range 2 {+ f.poll(t, "pbs-main")+ if !reflect.DeepEqual(before, f.points(t, "pbs-main")) || !reflect.DeepEqual(raw, f.m.state.GetSnapshot().PBSBackups) {+ t.Error("unread datastore scope deleted or renewed accepted artifacts")+ }+ if f.backupReads.Load() != reads || !f.m.pbsBackupCacheTimeFor("pbs-main", key).Equal(cacheAt) {+ t.Error("unread datastore scope guessed a child read or renewed cache")+ }+ for _, p := range f.postures(t) {+ if p.State == recovery.ProtectionStateProtected || len(p.ProviderStates) != 1 {+ t.Errorf("unread scope kept protection: %+v", p)+ continue+ }+ q := p.ProviderStates[0]+ if q.HistoryCompleteness != recovery.ProtectionHistoryUnavailable || q.Permissions != operationaltrust.EvidencePermissionsUnknown || q.JobState != recovery.OutcomeFailed {+ t.Errorf("unread scope lost uncertainty: %+v", q)+ }+ }+ inst := pbsInstanceByName(t, f.m.state.GetSnapshot(), "pbs-main")+ if inst.Status != "online" || inst.ConnectionHealth != "healthy" {+ t.Error("unread backup history was confused with connectivity")+ }+ }+ f.datastoreBody.Store(nil)+ f.datastoreStatus.Store(0)+ f.backupAt.Add(60)+ f.poll(t, "pbs-main")+ for _, p := range f.postures(t) {+ if p.State != recovery.ProtectionStateProtected {+ t.Errorf("ordinary recovery did not restore protection: %+v", p)+ }+ }+ empty := `{"data":[],"errors":null}`+ f.datastoreBody.Store(&empty)+ reads = f.backupReads.Load()+ f.poll(t, "pbs-main")+ if len(f.points(t, "pbs-main")) != 0 || len(f.m.state.GetSnapshot().PBSBackups) != 0 || f.backupReads.Load() != reads || !f.m.pbsBackupCacheTimeFor("pbs-main", key).IsZero() {+ t.Error("complete empty scope did not reconcile genuine absence")+ }+ })+ }+ }+ }+}++// Export the actual public poll -> worker -> live/store/posture results for+// consuming-surface evidence. This does not introduce a second production path.+func TestPBSDatastoreWholeResponseConsumerLifecycle(t *testing.T) {+ for _, kind := range []string{"vm", "ct"} {+ t.Run(kind, func(t *testing.T) {+ f := newPBSNamespacePollFixture(t, kind)+ frames := map[string]any{}+ var prior map[string]recovery.RecoveryPoint+ var raw []models.PBSBackup+ for _, phase := range []string{"observed", "unread-datastores", "recovered", "confirmed-empty"} {+ switch phase {+ case "unread-datastores":+ bad := `{"data":[{"store":"archive"}],"data":[],"errors":{"private":"fixture-only"}}`+ f.datastoreBody.Store(&bad)+ case "confirmed-empty":+ empty := `{"data":[]}`+ f.datastoreBody.Store(&empty)+ default:+ f.datastoreBody.Store(nil)+ }+ f.poll(t, "pbs-main")+ points, rows := f.points(t, "pbs-main"), f.m.state.GetSnapshot().PBSBackups+ if phase == "unread-datastores" && (!reflect.DeepEqual(prior, points) || !reflect.DeepEqual(raw, rows)) {+ t.Fatal("unread response removed or renewed backup evidence")+ }+ for _, p := range f.postures(t) {+ if (phase == "observed" || phase == "recovered") != (p.State == recovery.ProtectionStateProtected) {+ t.Fatalf("phase %s posture=%+v", phase, p)+ }+ }+ want := 2+ if phase == "confirmed-empty" {+ want = 0+ }+ if len(points) != want || len(rows) != want {+ t.Fatalf("phase %s lost or invented inventory", phase)+ }+ frames[phase] = map[string]any{"pbs": map[string]any{"data": map[string]any{"backups": rows}}, "postures": map[string]any{"data": f.postures(t), "policy": recovery.DefaultProtectionPosturePolicy.Payload()}, "points": points}+ prior, raw = points, rows+ }+ if dest := os.Getenv("PULSE_PBS_DATASTORE_FRAMES"); dest != "" {+ hashes := map[string]string{}+ for _, file := range []string{"monitor_pbs_pmg.go", "monitor_backups.go", "backup_guard.go", "../../pkg/pbs/client.go", "../../pkg/pbs/backup_inventory_response.go", "../recovery/posture.go"} {+ content, err := os.ReadFile(file)+ if err != nil {+ t.Fatal(err)+ }+ hashes[file] = fmt.Sprintf("%x", sha256.Sum256(content))+ }+ data, err := json.Marshal(map[string]any{"runtime_sha256": hashes, "resource_ids": f.ids, "frames": frames})+ if err != nil {+ t.Fatal(err)+ }+ if err := os.WriteFile(dest+"/frames-"+kind+".json", data, 0600); err != nil {+ t.Fatal(err)+ }+ }+ })+ }+}diff --git a/internal/monitoring/pbs_protection_observation_test.go b/internal/monitoring/pbs_protection_observation_test.goindex 9908aeab7e..fc6ab19347 100644--- a/internal/monitoring/pbs_protection_observation_test.go+++ b/internal/monitoring/pbs_protection_observation_test.go@@ -314,18 +314,31 @@ type pbsNamespacePollFixture struct {ids []string}-func newPBSNamespacePollFixture(t *testing.T) *pbsNamespacePollFixture {+func newPBSNamespacePollFixture(t *testing.T, kinds ...string) *pbsNamespacePollFixture {t.Helper()m, manager := recoveryIngestTestMonitor(t)f := &pbsNamespacePollFixture{m: m}f.backupAt.Store(time.Now().UTC().Add(-time.Hour).Truncate(time.Second).Unix())m.state = models.NewState()- m.state.UpdateVMs([]models.VM{- {ID: makeGuestID("cluster", "node-a", 100), VMID: 100, Instance: "cluster", Node: "node-a", Name: "guest-100", Status: "running"},- {ID: makeGuestID("cluster", "node-b", 101), VMID: 101, Instance: "cluster", Node: "node-b", Name: "guest-101", Status: "running"},- })+ kind := "vm"+ if len(kinds) > 0 {+ kind = kinds[0]+ }+ resourceType := unifiedresources.ResourceTypeVM+ if kind == "ct" {+ resourceType = unifiedresources.ResourceTypeSystemContainer+ m.state.UpdateContainers([]models.Container{+ {ID: makeGuestID("cluster", "node-a", 100), VMID: 100, Instance: "cluster", Node: "node-a", Name: "guest-100", Status: "running"},+ {ID: makeGuestID("cluster", "node-b", 101), VMID: 101, Instance: "cluster", Node: "node-b", Name: "guest-101", Status: "running"},+ })+ } else {+ m.state.UpdateVMs([]models.VM{+ {ID: makeGuestID("cluster", "node-a", 100), VMID: 100, Instance: "cluster", Node: "node-a", Name: "guest-100", Status: "running"},+ {ID: makeGuestID("cluster", "node-b", 101), VMID: 101, Instance: "cluster", Node: "node-b", Name: "guest-101", Status: "running"},+ })+ }for _, id := range []int{100, 101} {- f.ids = append(f.ids, unifiedresources.ProxmoxGuestCanonicalID(unifiedresources.ResourceTypeVM, "cluster", id))+ f.ids = append(f.ids, unifiedresources.ProxmoxGuestCanonicalID(resourceType, "cluster", id))}var err errorf.store, err = manager.StoreForOrg("default")@@ -345,8 +358,11 @@ func newPBSNamespacePollFixture(t *testing.T) *pbsNamespacePollFixture {data = map[string]any{"version": "3.4.2"}case "/api2/json/admin/datastore":if status := f.datastoreStatus.Load(); status != 0 {- http.Error(w, "fixture discovery unavailable", int(status))- return+ if status >= 400 {+ http.Error(w, "fixture discovery unavailable", int(status))+ return+ }+ w.WriteHeader(int(status))}if body := f.datastoreBody.Load(); body != nil {_, _ = w.Write([]byte(*body))@@ -376,7 +392,7 @@ func newPBSNamespacePollFixture(t *testing.T) *pbsNamespacePollFixture {f.childReads.Add(1)}if id != "100" || !f.rootEmpty.Load() {- data = []map[string]any{{"backup-type": "vm", "backup-id": id, "last-backup": f.backupAt.Load(), "backup-count": 1}}+ data = []map[string]any{{"backup-type": kind, "backup-id": id, "last-backup": f.backupAt.Load(), "backup-count": 1}}}case "/api2/json/admin/datastore/archive/snapshots":f.backupReads.Add(1)@@ -384,7 +400,7 @@ func newPBSNamespacePollFixture(t *testing.T) *pbsNamespacePollFixture {if r.URL.Query().Get("ns") == "tenant" {f.childReads.Add(1)}- data = []map[string]any{{"backup-type": "vm", "backup-id": id, "backup-time": f.backupAt.Load(), "size": 1024,+ data = []map[string]any{{"backup-type": kind, "backup-id": id, "backup-time": f.backupAt.Load(), "size": 1024,"files": []string{"index.json.blob"}, "verification": map[string]any{"state": "ok"}, "comment": "guest-" + id}}}_ = json.NewEncoder(w).Encode(map[string]any{"data": data})diff --git a/pkg/pbs/backup_inventory_response.go b/pkg/pbs/backup_inventory_response.goindex 6017939bcc..2237a930f1 100644--- a/pkg/pbs/backup_inventory_response.go+++ b/pkg/pbs/backup_inventory_response.go@@ -16,6 +16,16 @@ var errInvalidBackupInventoryResponse = errors.New("invalid PBS backup inventory// unambiguous envelope from a fully read, bounded response, never a JSON prefix.// Keep provider payloads out of format diagnostics (including reported errors).func readBackupInventory[T any](resp *http.Response) ([]T, error) {+ body, err := readBackupInventoryBody(resp)+ if err != nil {+ return nil, err+ }+ return decodeBackupInventory[T](body)+}++// Datastore discovery needs the same transport admission, but retains its+// existing HTML/HTTPS guidance and payload-free byte-count diagnostics.+func readBackupInventoryBody(resp *http.Response) ([]byte, error) {body, readErr := readResponseBodyLimited(resp.Body)closeErr := resp.Body.Close()if readErr != nil || closeErr != nil {@@ -24,6 +34,10 @@ func readBackupInventory[T any](resp *http.Response) ([]T, error) {if resp.StatusCode != http.StatusOK || !utf8.Valid(body) {return nil, errInvalidBackupInventoryResponse}+ return body, nil+}++func decodeBackupInventory[T any](body []byte) ([]T, error) {decoder := json.NewDecoder(bytes.NewReader(body))start, err := decoder.Token()if err != nil || start != json.Delim('{') {diff --git a/pkg/pbs/client.go b/pkg/pbs/client.goindex 323df4fb4f..8017e77b79 100644--- a/pkg/pbs/client.go+++ b/pkg/pbs/client.go@@ -903,11 +903,9 @@ func (c *Client) GetDatastoresFiltered(if err != nil {return nil, err}- defer resp.Body.Close()-- body, err := readResponseBodyLimited(resp.Body)+ body, err := readBackupInventoryBody(resp)if err != nil {- return nil, fmt.Errorf("failed to read response: %w", err)+ return nil, fmt.Errorf("failed to read datastore inventory: %w", err)}// Check if response is HTML (error page) instead of JSON@@ -919,14 +917,12 @@ func (c *Client) GetDatastoresFiltered(return nil, fmt.Errorf("PBS returned HTML instead of JSON (likely an error page). Please check your PBS URL and port (default is 8007)")}- var datastoreList struct {- Data []struct {- Store string `json:"store"`- Comment string `json:"comment,omitempty"`- } `json:"data"`+ type datastoreEntry struct {+ Store string `json:"store"`+ Comment string `json:"comment,omitempty"`}-- if err := json.Unmarshal(body, &datastoreList); err != nil {+ datastoreList, err := decodeBackupInventory[datastoreEntry](body)+ if err != nil {log.Error().Int("responseBytes", len(body)).Err(err).@@ -936,10 +932,7 @@ func (c *Client) GetDatastoresFiltered(// Only an explicit array establishes the datastore scope. In particular,// missing/null data and nameless rows are not an authoritative empty list:// callers use a successful empty enumeration to reconcile backup history.- if datastoreList.Data == nil {- return nil, fmt.Errorf("datastore list response contains no data array")- }- for _, ds := range datastoreList.Data {+ for _, ds := range datastoreList {if strings.TrimSpace(ds.Store) == "" {return nil, fmt.Errorf("datastore list response contains an unnamed datastore")}@@ -947,7 +940,7 @@ func (c *Client) GetDatastoresFiltered(// Now get status for each datastorevar datastores []Datastore- for _, ds := range datastoreList.Data {+ for _, ds := range datastoreList {if include != nil && !include(ds.Store) {continue}diff --git a/pkg/pbs/client_http_test.go b/pkg/pbs/client_http_test.goindex 7bb1c3ec68..17f7cd26c9 100644--- a/pkg/pbs/client_http_test.go+++ b/pkg/pbs/client_http_test.go@@ -1253,3 +1253,129 @@ func TestClient_ListRunningDataTasksRequiresHTTP200(t *testing.T) {}}}++// Datastore names establish the outer scope used to reconcile every backup.+// Validate the complete envelope before applying exclusions or reading children.+func TestDatastoreInventoryRequiresWholeResponse(t *testing.T) {+ for name, body := range map[string]string{+ "second value": `{"data":[]} {"data":[]}`,+ "trailing garbage": `{"data":[]} private-provider-detail`,+ "reported error": `{"data":[],"errors":{"private":"private-provider-detail"}}`,+ "empty errors": `{"data":[],"errors":{}}`,+ "duplicate data": `{"data":null,"data":[]}`,+ "discarded datastore": `{"data":[{"store":"archive"}],"data":[]}`,+ "escaped duplicate": `{"data":null,"d\u0061ta":[]}`,+ "duplicate errors": `{"data":[],"errors":{"private":"private-provider-detail"},"errors":null}`,+ "case data": `{"data":null,"DATA":[]}`,+ "case errors": `{"data":[],"errors":{"private":"private-provider-detail"},"Errors":null}`,+ "duplicate metadata": `{"data":[],"total":1,"total":0}`,+ "invalid utf8": "{\"data\":[],\"metadata\":\"\xff\"}",+ } {+ for _, excluded := range []bool{false, true} {+ suffix := "/included"+ if excluded {+ suffix = "/excluded"+ }+ t.Run(name+suffix, func(t *testing.T) {+ reads, filtered := 0, 0+ client, server := newTestClient(t, func(w http.ResponseWriter, r *http.Request) {+ reads+++ if r.Method != http.MethodGet || r.URL.Path != "/api2/json/admin/datastore" {+ t.Errorf("unread discovery caused a child or mutation request: %s %s", r.Method, r.URL)+ }+ _, _ = w.Write([]byte(body))+ })+ defer server.Close()+ rows, err := client.GetDatastoresFiltered(context.Background(), func(string) bool { filtered++; return !excluded })+ if err == nil || rows != nil {+ t.Fatalf("unread datastore scope accepted: rows=%+v err=%v", rows, err)+ }+ if strings.Contains(err.Error(), "private-provider-detail") {+ t.Error("provider text leaked into format error")+ }+ if reads != 1 || filtered != 0 {+ t.Errorf("discovery failed after side effects: reads=%d filters=%d", reads, filtered)+ }+ })+ }+ }+}++func TestDatastoreInventoryRequiresCompleteHTTP200Body(t *testing.T) {+ for _, failure := range []string{"read after JSON", "close", "size", "created", "accepted", "partial"} {+ t.Run(failure, func(t *testing.T) {+ body := &inventoryResponseBody{Reader: strings.NewReader(`{"data":[]}`)}+ status := http.StatusOK+ switch failure {+ case "read after JSON":+ body.Reader = io.MultiReader(body.Reader, inventoryReadFailure{})+ case "close":+ body.closeErr = errors.New("fixture close failure")+ case "size":+ body.Reader = io.MultiReader(body.Reader, strings.NewReader(strings.Repeat(" ", int(maxResponseBodyBytes))))+ case "created":+ status = http.StatusCreated+ case "accepted":+ status = http.StatusAccepted+ case "partial":+ status = http.StatusPartialContent+ }+ client, err := NewClient(ClientConfig{Host: "https://fixture.invalid", TokenName: "fixture@pbs!read", TokenValue: "fixture-only"})+ if err != nil {+ t.Fatal(err)+ }+ reads := 0+ client.httpClient = &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {+ reads+++ return &http.Response{StatusCode: status, Body: body, Header: make(http.Header)}, nil+ })}+ rows, err := client.GetDatastores(context.Background())+ if err == nil || rows != nil {+ t.Errorf("incomplete datastore response accepted: rows=%+v err=%v", rows, err)+ }+ if reads != 1 || !body.closed || int64(body.bytes) > maxResponseBodyBytes+1 {+ t.Errorf("reads=%d closed=%t bytes=%d", reads, body.closed, body.bytes)+ }+ })+ }+}++func TestDatastoreInventoryAcceptsCompleteScopeBeforeFiltering(t *testing.T) {+ for _, empty := range []bool{false, true} {+ t.Run(map[bool]string{false: "included-and-excluded", true: "empty"}[empty], func(t *testing.T) {+ requests := []string{}+ client, server := newTestClient(t, func(w http.ResponseWriter, r *http.Request) {+ requests = append(requests, r.URL.RequestURI())+ switch r.URL.Path {+ case "/api2/json/admin/datastore":+ data := []map[string]string{}+ if !empty {+ data = []map[string]string{{"store": "archive", "comment": "literal <not markup>"}, {"store": "excluded"}}+ }+ _ = json.NewEncoder(w).Encode(map[string]any{"data": data, "errors": nil, "metadata": map[string]any{"future": "温度"}})+ case "/api2/json/admin/datastore/archive/rrd":+ _, _ = w.Write([]byte(`{"data":[]}`))+ case "/api2/json/admin/datastore/archive/status":+ _, _ = w.Write([]byte(`{"data":{"total":100,"used":40,"avail":60,"deduplication-factor":2}}`))+ default:+ t.Errorf("unexpected child read: %s", r.URL)+ http.NotFound(w, r)+ }+ })+ defer server.Close()+ rows, err := client.GetDatastoresFiltered(context.Background(), func(name string) bool { return name != "excluded" })+ if err != nil {+ t.Fatal(err)+ }+ if empty {+ if len(rows) != 0 || len(requests) != 1 {+ t.Fatalf("empty response rows=%+v requests=%v", rows, requests)+ }+ return+ }+ if len(rows) != 1 || rows[0].Store != "archive" || rows[0].Status != "available" || rows[0].Total != 100 || rows[0].Used != 40 || rows[0].Avail != 60 || rows[0].DeduplicationFactor != 2 || len(requests) != 3 {+ t.Fatalf("ordinary read/filter changed: rows=%+v requests=%v", rows, requests)+ }+ })+ }+}