Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Keep failed optional PMG reads from throttling the connection
rcourtman0d7f2b7997da
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex 62197dd277..4e0cf6f55b 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -832,13 +832,32 @@ four switches; pausing the connection prevents every subsequent ordinary poll.An edit or pause does not revoke an already running poll's snapshot.A successful version read still establishes online connection health, not-complete collection. Downstream read failures are retained as failed poll+complete collection. Downstream read failures are retained as failed readevidence, with missing fields instead of synthetic zero readings. Quarantinetotals require both category reads. Collector controls in `monitor_pmg_test.go`cover sixteen scopes, next-poll edits/pause/resume, in-flight snapshots, eachread failure, recovery, legacy defaults and cancelled/unknown connections.These synthetic controls are not native PMG or installed privacy acceptance.+Only connection, authentication and version failures, and a poll cut short by+its own deadline or cancellation, are scheduler failures that drive backoff,+the circuit breaker and the dead-letter queue. A downstream read that fails+after the connection succeeded goes through `recordTaskOutcome` as a read+error: the poll keeps its normal interval and freshness, and the error stays+the connection's outstanding poll error (category `partial`) until a poll+completes every read. Before this split one broken optional endpoint (the+queue read in #2960) dead-lettered the whole connection to a 30-minute poll+and left every other PMG reading stale. PBS already keeps its optional+datastore and node-status reads out of the scheduler outcome.+`TestPMGOptionalReadFailureKeepsNormalSchedule`,+`TestPMGVersionFailureStillBacksOffAndDeadLetters`,+`TestPMGCancelledPollWithFailedReadsBacksOff` and+`TestPBSOptionalReadFailureKeepsNormalSchedule` in+`partial_read_scheduling_test.go` drive the poll and reschedule steps together;+`TestRecordTaskOutcomeReadErrorSchedulesAsSuccess` in+`monitor_additional_test.go` pins the scheduler bookkeeping, which records the+success and the outstanding read error in one update.+The PMG client rejects omitted/null object payloads for version, mail, queueand quarantine instead of materialising them as zero structs. Explicitsuccessful zero objects remain observations. `TestPMGCollectionMissingObjectData`@@ -854,7 +873,7 @@ the connection name as the backup node only when PMG answered with nocluster entries; after a failed cluster read that name is a display label,and requesting backups with it produced a misleading hostname-lookup errorwithout ever returning backups. The poll still records the cluster read-failure. It requests no backups, and the connection's last-known backups+failure as a read error. It requests no backups, and the connection's last-known backupsare kept rather than replaced by an empty list (see the next section).`TestClientClusterStatusIgnoresClusterIDFormat` and the`TestPollPMGInstance*` issue tests in `monitor_pmg_test.go` pin all three@@ -869,7 +888,7 @@ read, a node whose backup read failed, and a node a cancelled poll neverreached. This is the same per-scope carry that `preserveFailedStorageBackups`applies to failed PVE storages. A node that answered with no backups, or thata successful cluster read no longer lists, still clears its entries. The-failed read is still recorded as a poll failure, and the connection-name+failed read is still recorded as a read error, and the connection-namefallback above is unchanged. `TestPMGBackupsSurviveFailedClusterRead`,`TestPMGBackupsSurviveEveryNodeBackupReadFailing`,`TestPMGBackupsKeepFailedNodeOnPartialRead` anddiff --git a/internal/monitoring/monitor.go b/internal/monitoring/monitor.goindex 4f44505d00..b7fe3f44dd 100644--- a/internal/monitoring/monitor.go+++ b/internal/monitoring/monitor.go@@ -3043,7 +3043,22 @@ func (m *Monitor) ensureBreaker(key string) *circuitBreaker {return breaker}+// pollReadErrorCategory marks a poll status error left by a downstream read+// that failed after the connection itself succeeded.+const pollReadErrorCategory = "partial"+func (m *Monitor) recordTaskResult(instanceType InstanceType, instance string, pollErr error) {+ m.recordTaskOutcome(instanceType, instance, pollErr, nil)+}++// recordTaskOutcome records one poll for the scheduler. pollErr is a+// connection, authentication, version or whole-poll failure: it counts toward+// backoff, the circuit breaker and the dead-letter queue. readErr is a+// downstream read that failed after the connection succeeded. It schedules the+// poll as a success, so one broken optional endpoint cannot throttle every+// other reading of the connection (#2960), but stays the connection's+// outstanding poll error until a poll completes every read.+func (m *Monitor) recordTaskOutcome(instanceType InstanceType, instance string, pollErr, readErr error) {if m == nil {return}@@ -3083,6 +3098,11 @@ func (m *Monitor) recordTaskResult(instanceType InstanceType, instance string, pstatus.LastErrorAt = time.Time{}status.LastErrorMessage = ""status.LastErrorCategory = ""+ if readErr != nil {+ status.LastErrorAt = now+ status.LastErrorMessage = readErr.Error()+ status.LastErrorCategory = pollReadErrorCategory+ }m.mu.Unlock()if breaker != nil {breaker.recordSuccess()diff --git a/internal/monitoring/monitor_additional_test.go b/internal/monitoring/monitor_additional_test.goindex 25058ff5bc..dd78c7cbb1 100644--- a/internal/monitoring/monitor_additional_test.go+++ b/internal/monitoring/monitor_additional_test.go@@ -1882,6 +1882,45 @@ func TestMonitorPersistentFailureBackoffAndRecovery(t *testing.T) {}}+// A downstream read failure after a successful connection is a scheduler+// success that keeps the read error outstanding in one update (#2960).+func TestRecordTaskOutcomeReadErrorSchedulesAsSuccess(t *testing.T) {+ m := &Monitor{+ config: &config.Config{},+ circuitBreakers: make(map[string]*circuitBreaker),+ pollStatusMap: make(map[string]*pollStatus),+ failureCounts: make(map[string]int),+ lastOutcome: make(map[string]taskOutcome),+ }+ task := ScheduledTask{InstanceType: InstanceTypePMG, InstanceName: "gateway"}+ key := schedulerKey(task.InstanceType, task.InstanceName)+ for i := 0; i < 3; i++ {+ m.recordTaskResult(task.InstanceType, task.InstanceName, errors.New("connection timeout"))+ }+ if m.allowExecution(task) {+ t.Fatal("connection failures did not open the breaker")+ }++ m.recordTaskOutcome(task.InstanceType, task.InstanceName, nil, errors.New("pmg_queue_status: decode failed"))+ if outcome := m.lastOutcome[key]; !outcome.success || outcome.err != nil || m.failureCounts[key] != 0 {+ t.Fatalf("read error scheduled as a failure: outcome=%+v failures=%d", outcome, m.failureCounts[key])+ }+ if !m.allowExecution(task) || len(m.SchedulerHealth().Breakers) != 0 {+ t.Fatal("read error kept the breaker open")+ }+ status := m.pollStatusMap[key]+ if status.ConsecutiveFailures != 0 || status.LastSuccess.IsZero() || !status.FirstFailureAt.IsZero() ||+ status.LastErrorCategory != pollReadErrorCategory || !strings.Contains(status.LastErrorMessage, "pmg_queue_status") ||+ !status.LastErrorAt.Equal(status.LastSuccess) {+ t.Fatalf("read error not kept as the outstanding poll error: %+v", status)+ }++ m.recordTaskOutcome(task.InstanceType, task.InstanceName, nil, nil)+ if status.LastErrorMessage != "" || status.LastErrorCategory != "" || !status.LastErrorAt.IsZero() {+ t.Fatalf("complete poll left the read error outstanding: %+v", status)+ }+}+func TestGuestConfigPlacement(t *testing.T) {for _, kind := range []string{"vm", "container", "lxc"} {for _, reverse := range []bool{false, true} {diff --git a/internal/monitoring/monitor_pbs_pmg.go b/internal/monitoring/monitor_pbs_pmg.goindex f0f9448bd3..1488592cee 100644--- a/internal/monitoring/monitor_pbs_pmg.go+++ b/internal/monitoring/monitor_pbs_pmg.go@@ -697,7 +697,10 @@ func (m *Monitor) pollPMGInstance(ctx context.Context, instanceName string, cliestart := time.Now()debugEnabled := logging.IsLevelEnabled(zerolog.DebugLevel)- var pollErr error+ // pollErr is a connection, version or whole-poll failure and drives+ // scheduler backoff. readErr collects downstream reads that failed after+ // the connection succeeded; see recordTaskOutcome.+ var pollErr, readErr errorif m.pollMetrics != nil {m.pollMetrics.IncInFlight("pmg")defer m.pollMetrics.DecInFlight("pmg")@@ -722,7 +725,7 @@ func (m *Monitor) pollPMGInstance(ctx context.Context, instanceName string, clie}()}defer func() {- m.recordTaskResult(InstanceTypePMG, instanceName, pollErr)+ m.recordTaskOutcome(InstanceTypePMG, instanceName, pollErr, readErr)}()select {@@ -806,10 +809,11 @@ func (m *Monitor) pollPMGInstance(ctx context.Context, instanceName string, cliem.resetAuthFailures(instanceName, "pmg")// A successful version read establishes connection health, not complete- // downstream collection. Preserve missing readings and report failed polls.+ // downstream collection. Preserve missing readings and report failed reads+ // without backing off the readings that succeeded.recordReadError := func(operation string, err error) {if err != nil {- pollErr = stderrors.Join(pollErr, errors.WrapAPIError(operation, instanceName, err, 0))+ readErr = stderrors.Join(readErr, errors.WrapAPIError(operation, instanceName, err, 0))}}@@ -922,7 +926,7 @@ func (m *Monitor) pollPMGInstance(ctx context.Context, instanceName string, clie// Only a node whose backup list was read is authoritative this cycle. A// failed cluster read, a failed per-node read or a cancelled poll observed// nothing, so those nodes keep their last-known backups instead of being- // published as having none; the failed read is already in pollErr.+ // published as having none; the failed read is already in readErr.if clusterErr != nil || len(readBackupNodes) < len(backupNodes) {var preservedBackupNodes []stringpmgBackups, preservedBackupNodes = preserveUnreadPMGNodeBackups(@@ -1078,6 +1082,12 @@ func (m *Monitor) pollPMGInstance(ctx context.Context, instanceName string, clie}}+ // Reads cut short by the poll's own deadline or cancellation are a+ // whole-poll failure, not one broken endpoint, and keep their backoff.+ if ctxErr := ctx.Err(); ctxErr != nil && readErr != nil {+ pollErr = stderrors.Join(ctxErr, readErr)+ }+m.state.UpdatePMGBackups(instanceName, pmgBackups)m.state.UpdatePMGInstance(pmgInst)log.Info().diff --git a/internal/monitoring/monitor_pmg_test.go b/internal/monitoring/monitor_pmg_test.goindex 14a7a7070e..96def51631 100644--- a/internal/monitoring/monitor_pmg_test.go+++ b/internal/monitoring/monitor_pmg_test.go@@ -512,16 +512,21 @@ func TestPMGCollectionFailureEvidence(t *testing.T) {f.fail = opm.pollPMGInstance(context.Background(), "gateway", f)status := m.pollStatusMap["pmg::gateway"]- if status.ConsecutiveFailures != 1 || status.LastErrorMessage == "" || !status.LastSuccess.Equal(before) {- t.Fatalf("failed read recorded as complete: %+v", status)- }v := m.state.GetSnapshot().PMGInstances[0]if op == "version" {+ if status.ConsecutiveFailures != 1 || status.LastErrorMessage == "" || !status.LastSuccess.Equal(before) {+ t.Fatalf("failed connection recorded as complete: %+v", status)+ }if v.Status != "offline" || v.ConnectionHealth != "unhealthy" {t.Fatal("failed connection not offline")}return}+ // A downstream read failure stays visible on the connection but+ // is not a scheduler failure (#2960).+ if status.ConsecutiveFailures != 0 || status.LastErrorMessage == "" || status.LastErrorCategory != pollReadErrorCategory || status.LastSuccess.Before(before) {+ t.Fatalf("failed read recorded as complete or as a connection failure: %+v", status)+ }if v.Status != "online" || v.ConnectionHealth != "healthy" || !m.state.GetSnapshot().ConnectionHealth["pmg-gateway"] {t.Fatal("downstream failure mislabeled as version connection failure")}@@ -714,11 +719,11 @@ func pollPMGBackupRetention(t *testing.T, m *Monitor, f *pmgBackupRetentionFixtum.pollPMGInstance(context.Background(), "gateway", f)}-func assertPMGPollFailed(t *testing.T, m *Monitor) {+func assertPMGReadFailed(t *testing.T, m *Monitor) {t.Helper()status := m.pollStatusMap["pmg::gateway"]- if status == nil || status.ConsecutiveFailures == 0 || status.LastErrorMessage == "" {- t.Fatalf("failed backup enumeration not recorded as a poll failure: %+v", status)+ if status == nil || status.LastErrorMessage == "" || status.LastErrorCategory != pollReadErrorCategory {+ t.Fatalf("failed backup enumeration not recorded as a failed read: %+v", status)}}@@ -742,7 +747,7 @@ func TestPMGBackupsSurviveFailedClusterRead(t *testing.T) {if got := pmgBackupFilesByNode(t, m); !reflect.DeepEqual(got, want) {t.Fatalf("failed cluster read replaced backups: got=%v want=%v", got, want)}- assertPMGPollFailed(t, m)+ assertPMGReadFailed(t, m)}// When every per-node backup read fails, nothing was observed this poll and@@ -759,7 +764,7 @@ func TestPMGBackupsSurviveEveryNodeBackupReadFailing(t *testing.T) {if got := pmgBackupFilesByNode(t, m); !reflect.DeepEqual(got, want) {t.Fatalf("failed backup reads replaced backups: got=%v want=%v", got, want)}- assertPMGPollFailed(t, m)+ assertPMGReadFailed(t, m)}// A failed read on one node keeps that node's last-known backups while the@@ -776,7 +781,7 @@ func TestPMGBackupsKeepFailedNodeOnPartialRead(t *testing.T) {if got := pmgBackupFilesByNode(t, m); !reflect.DeepEqual(got, want) {t.Fatalf("partial read backups=%v want=%v", got, want)}- assertPMGPollFailed(t, m)+ assertPMGReadFailed(t, m)}// A complete enumeration is authoritative: nodes that answered with nodiff --git a/internal/monitoring/partial_read_scheduling_test.go b/internal/monitoring/partial_read_scheduling_test.gonew file mode 100644index 0000000000..0eb01e7d2e--- /dev/null+++ b/internal/monitoring/partial_read_scheduling_test.go@@ -0,0 +1,222 @@+package monitoring++import (+ "context"+ "fmt"+ "net/http"+ "net/http/httptest"+ "strings"+ "testing"+ "time"++ "github.com/rcourtman/pulse-go-rewrite/internal/config"+ "github.com/rcourtman/pulse-go-rewrite/pkg/pbs"+)++// pollAndReschedule runs one poll followed by the scheduler's reschedule step,+// as the worker loop does, and returns the task the scheduler queued next.+func pollAndReschedule(t *testing.T, m *Monitor, task ScheduledTask, poll func()) ScheduledTask {+ t.Helper()+ poll()+ m.rescheduleTask(task)+ next, ok := m.taskQueue.Get(task.InstanceType, task.InstanceName)+ if !ok {+ if m.deadLetterQueue.Size() == 0 {+ t.Fatal("poll was neither rescheduled nor dead-lettered")+ }+ return ScheduledTask{}+ }+ return next+}++func newPartialReadPMGMonitor(t *testing.T) (*Monitor, ScheduledTask) {+ t.Helper()+ cfg := allPMGScope()+ cfg.Name = "gateway"+ m, err := New(&config.Config{PMGInstances: []config.PMGInstance{cfg}})+ if err != nil {+ t.Fatalf("failed to create monitor: %v", err)+ }+ interval := m.baseIntervalForInstanceType(InstanceTypePMG)+ if interval <= 0 {+ t.Fatalf("no PMG base interval: %v", interval)+ }+ return m, ScheduledTask{InstanceName: "gateway", InstanceType: InstanceTypePMG, Interval: interval, NextRun: time.Now()}+}++// Issue #2960: one failed optional PMG read must not back off or dead-letter+// the connection. The failed reading stays missing and the error stays on the+// connection, while every other reading keeps its normal cadence.+func TestPMGOptionalReadFailureKeepsNormalSchedule(t *testing.T) {+ for _, op := range []string{"cluster", "backup/one", "queue/one", "mail", "count", "scores", "quarantine/spam", "quarantine/virus", "domains", "domain-stats"} {+ t.Run(op, func(t *testing.T) {+ m, task := newPartialReadPMGMonitor(t)+ f := &pmgScopeFixture{calls: map[string]int{}, fail: op}++ for i := 0; i <= m.maxRetryAttempts+1; i++ {+ next := pollAndReschedule(t, m, task, func() { m.pollPMGInstance(context.Background(), "gateway", f) })+ if next.Interval != task.Interval {+ t.Fatalf("poll %d: interval=%v, want normal %v", i, next.Interval, task.Interval)+ }+ task = next+ }+ if size := m.deadLetterQueue.Size(); size != 0 {+ t.Fatalf("optional read failure reached the dead-letter queue (%d tasks)", size)+ }+ if state, failures, _ := m.ensureBreaker(schedulerKey(InstanceTypePMG, "gateway")).State(); state != "closed" || failures != 0 {+ t.Fatalf("breaker=%s failures=%d after optional read failures", state, failures)+ }++ status := m.pollStatusMap["pmg::gateway"]+ if status.ConsecutiveFailures != 0 || status.LastSuccess.IsZero() {+ t.Fatalf("partial poll counted as a connection failure: %+v", status)+ }+ if status.LastErrorMessage == "" || status.LastErrorCategory != pollReadErrorCategory {+ t.Fatalf("failed read not left visible on the connection: %+v", status)+ }++ v := m.state.GetSnapshot().PMGInstances[0]+ if v.Status != "online" || v.ConnectionHealth != "healthy" {+ t.Fatalf("partial poll rewrote connection health: %s/%s", v.Status, v.ConnectionHealth)+ }+ nodeQueue := func(i int) bool { return len(v.Nodes) > i && v.Nodes[i].QueueStatus != nil }+ published := map[string]bool{+ "mail": v.MailStats != nil,+ "count": len(v.MailCount) > 0,+ "scores": len(v.SpamDistribution) > 0,+ "quarantine": v.Quarantine != nil,+ "domains": len(v.RelayDomains) > 0,+ "domain-stats": len(v.DomainStats) > 0,+ "queue/one": nodeQueue(0),+ "queue/two": nodeQueue(1),+ }+ failed := map[string]bool{op: true}+ switch {+ case op == "cluster":+ // Node queues are read per discovered node.+ failed = map[string]bool{"queue/one": true, "queue/two": true}+ case strings.HasPrefix(op, "quarantine/"):+ // Totals require both categories.+ failed = map[string]bool{"quarantine": true}+ }+ for reading, ok := range published {+ if failed[reading] && ok {+ t.Errorf("failed %s reading was published", reading)+ }+ if !failed[reading] && !ok {+ t.Errorf("%s reading was not published beside the failed %s read", reading, op)+ }+ }++ // A complete poll clears the outstanding read error.+ f.fail = ""+ m.pollPMGInstance(context.Background(), "gateway", f)+ if status.LastErrorMessage != "" || status.LastErrorCategory != "" {+ t.Fatalf("complete poll left the read error: %+v", status)+ }+ })+ }+}++// Connection failures keep the scheduler's backoff and dead-letter handling.+func TestPMGVersionFailureStillBacksOffAndDeadLetters(t *testing.T) {+ m, task := newPartialReadPMGMonitor(t)+ f := &pmgScopeFixture{calls: map[string]int{}, fail: "version"}++ next := pollAndReschedule(t, m, task, func() { m.pollPMGInstance(context.Background(), "gateway", f) })+ if next.Interval == task.Interval {+ t.Fatalf("version failure kept the normal interval %v", next.Interval)+ }+ if status := m.pollStatusMap["pmg::gateway"]; status.ConsecutiveFailures != 1 || status.LastErrorCategory == pollReadErrorCategory {+ t.Fatalf("version failure not recorded as a connection failure: %+v", status)+ }+ for i := 0; i < m.maxRetryAttempts && m.deadLetterQueue.Size() == 0; i++ {+ pollAndReschedule(t, m, task, func() { m.pollPMGInstance(context.Background(), "gateway", f) })+ }+ if m.deadLetterQueue.Size() != 1 {+ t.Fatal("repeated version failures never reached the dead-letter queue")+ }+}++// Reads cut short by the poll's own cancellation are a whole-poll failure,+// not one broken endpoint, and keep their transient backoff.+func TestPMGCancelledPollWithFailedReadsBacksOff(t *testing.T) {+ m, task := newPartialReadPMGMonitor(t)+ ctx, cancel := context.WithCancel(context.Background())+ defer cancel()+ f := &pmgScopeFixture{calls: map[string]int{}, fail: "mail", onVersion: cancel}++ next := pollAndReschedule(t, m, task, func() { m.pollPMGInstance(ctx, "gateway", f) })+ if next.Interval == task.Interval || m.deadLetterQueue.Size() != 0 {+ t.Fatalf("cancelled poll interval=%v dlq=%d, want transient backoff", next.Interval, m.deadLetterQueue.Size())+ }+ if status := m.pollStatusMap["pmg::gateway"]; status.ConsecutiveFailures != 1 {+ t.Fatalf("cancelled poll not recorded as a failure: %+v", status)+ }+}++// PBS keeps optional read failures (datastores, node status) out of the+// scheduler outcome already; pin that so a future change cannot reintroduce+// the PMG coupling behind #2960.+func TestPBSOptionalReadFailureKeepsNormalSchedule(t *testing.T) {+ for _, failing := range []string{"/api2/json/admin/datastore", "/api2/json/nodes/localhost/status"} {+ t.Run(strings.TrimPrefix(failing, "/api2/json/"), func(t *testing.T) {+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ if r.URL.Path == failing {+ w.WriteHeader(http.StatusInternalServerError)+ fmt.Fprint(w, `{"message":"fixture read refused","data":null}`)+ return+ }+ switch r.URL.Path {+ case "/api2/json/version":+ fmt.Fprint(w, `{"data":{"version":"4.0.1","release":"1","repoid":"fixture"}}`)+ case "/api2/json/nodes/localhost/status":+ fmt.Fprint(w, `{"data":{"cpu":0.25,"memory":{"used":512,"total":1024},"uptime":100}}`)+ case "/api2/json/admin/datastore":+ fmt.Fprint(w, `{"data":[]}`)+ default:+ http.NotFound(w, r)+ }+ }))+ t.Cleanup(server.Close)+ client, err := pbs.NewClient(pbs.ClientConfig{Host: server.URL, TokenName: "fixture@pbs!reader", TokenValue: "fixture-only"})+ if err != nil {+ t.Fatal(err)+ }+ m, err := New(&config.Config{PBSInstances: []config.PBSInstance{{+ Name: "pbs-main", Host: server.URL, TokenName: "fixture@pbs!reader", TokenValue: "fixture-only", MonitorDatastores: true,+ }}})+ if err != nil {+ t.Fatalf("failed to create monitor: %v", err)+ }+ task := ScheduledTask{InstanceName: "pbs-main", InstanceType: InstanceTypePBS, Interval: m.baseIntervalForInstanceType(InstanceTypePBS), NextRun: time.Now()}++ for i := 0; i <= m.maxRetryAttempts+1; i++ {+ next := pollAndReschedule(t, m, task, func() { m.pollPBSInstance(context.Background(), "pbs-main", client) })+ if next.Interval != task.Interval {+ t.Fatalf("poll %d: interval=%v, want normal %v", i, next.Interval, task.Interval)+ }+ task = next+ }+ if size := m.deadLetterQueue.Size(); size != 0 {+ t.Fatalf("optional read failure reached the dead-letter queue (%d tasks)", size)+ }+ if status := m.pollStatusMap["pbs::pbs-main"]; status == nil || status.ConsecutiveFailures != 0 {+ t.Fatalf("optional read failure counted as a poll failure: %+v", status)+ }++ inst := pbsInstanceByName(t, m.state.GetSnapshot(), "pbs-main")+ if inst.Status != "online" || inst.Version != "4.0.1" {+ t.Fatalf("connection readings not published: %+v", inst)+ }+ nodeStatusFailed := failing == "/api2/json/nodes/localhost/status"+ if inst.NodeMetricsUnavailable != nodeStatusFailed {+ t.Fatalf("node metrics unavailable=%v with node status failed=%v", inst.NodeMetricsUnavailable, nodeStatusFailed)+ }+ if !nodeStatusFailed && inst.CPU != 0.25 {+ t.Fatalf("node CPU not published beside the failed datastore read: %v", inst.CPU)+ }+ })+ }+}