Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Keep failed optional PMG reads from throttling the connection

rcourtman0d7f2b7997da
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index 62197dd277..4e0cf6f55b 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -832,13 +832,32 @@ four switches; pausing the connection prevents every subsequent ordinary poll.
An edit or pause does not revoke an already running poll's snapshot.
A successful version read still establishes online connection health, not
-complete collection. Downstream read failures are retained as failed poll
+complete collection. Downstream read failures are retained as failed read
evidence, with missing fields instead of synthetic zero readings. Quarantine
totals require both category reads. Collector controls in `monitor_pmg_test.go`
cover sixteen scopes, next-poll edits/pause/resume, in-flight snapshots, each
read failure, recovery, legacy defaults and cancelled/unknown connections.
These synthetic controls are not native PMG or installed privacy acceptance.
+Only connection, authentication and version failures, and a poll cut short by
+its own deadline or cancellation, are scheduler failures that drive backoff,
+the circuit breaker and the dead-letter queue. A downstream read that fails
+after the connection succeeded goes through `recordTaskOutcome` as a read
+error: the poll keeps its normal interval and freshness, and the error stays
+the connection's outstanding poll error (category `partial`) until a poll
+completes every read. Before this split one broken optional endpoint (the
+queue read in #2960) dead-lettered the whole connection to a 30-minute poll
+and left every other PMG reading stale. PBS already keeps its optional
+datastore and node-status reads out of the scheduler outcome.
+`TestPMGOptionalReadFailureKeepsNormalSchedule`,
+`TestPMGVersionFailureStillBacksOffAndDeadLetters`,
+`TestPMGCancelledPollWithFailedReadsBacksOff` and
+`TestPBSOptionalReadFailureKeepsNormalSchedule` in
+`partial_read_scheduling_test.go` drive the poll and reschedule steps together;
+`TestRecordTaskOutcomeReadErrorSchedulesAsSuccess` in
+`monitor_additional_test.go` pins the scheduler bookkeeping, which records the
+success and the outstanding read error in one update.
+
The PMG client rejects omitted/null object payloads for version, mail, queue
and quarantine instead of materialising them as zero structs. Explicit
successful zero objects remain observations. `TestPMGCollectionMissingObjectData`
@@ -854,7 +873,7 @@ the connection name as the backup node only when PMG answered with no
cluster entries; after a failed cluster read that name is a display label,
and requesting backups with it produced a misleading hostname-lookup error
without ever returning backups. The poll still records the cluster read
-failure. It requests no backups, and the connection's last-known backups
+failure as a read error. It requests no backups, and the connection's last-known backups
are kept rather than replaced by an empty list (see the next section).
`TestClientClusterStatusIgnoresClusterIDFormat` and the
`TestPollPMGInstance*` issue tests in `monitor_pmg_test.go` pin all three
@@ -869,7 +888,7 @@ read, a node whose backup read failed, and a node a cancelled poll never
reached. This is the same per-scope carry that `preserveFailedStorageBackups`
applies to failed PVE storages. A node that answered with no backups, or that
a successful cluster read no longer lists, still clears its entries. The
-failed read is still recorded as a poll failure, and the connection-name
+failed read is still recorded as a read error, and the connection-name
fallback above is unchanged. `TestPMGBackupsSurviveFailedClusterRead`,
`TestPMGBackupsSurviveEveryNodeBackupReadFailing`,
`TestPMGBackupsKeepFailedNodeOnPartialRead` and
diff --git a/internal/monitoring/monitor.go b/internal/monitoring/monitor.go
index 4f44505d00..b7fe3f44dd 100644
--- a/internal/monitoring/monitor.go
+++ b/internal/monitoring/monitor.go
@@ -3043,7 +3043,22 @@ func (m *Monitor) ensureBreaker(key string) *circuitBreaker {
return breaker
}
+// pollReadErrorCategory marks a poll status error left by a downstream read
+// that failed after the connection itself succeeded.
+const pollReadErrorCategory = "partial"
+
func (m *Monitor) recordTaskResult(instanceType InstanceType, instance string, pollErr error) {
+ m.recordTaskOutcome(instanceType, instance, pollErr, nil)
+}
+
+// recordTaskOutcome records one poll for the scheduler. pollErr is a
+// connection, authentication, version or whole-poll failure: it counts toward
+// backoff, the circuit breaker and the dead-letter queue. readErr is a
+// downstream read that failed after the connection succeeded. It schedules the
+// poll as a success, so one broken optional endpoint cannot throttle every
+// other reading of the connection (#2960), but stays the connection's
+// outstanding poll error until a poll completes every read.
+func (m *Monitor) recordTaskOutcome(instanceType InstanceType, instance string, pollErr, readErr error) {
if m == nil {
return
}
@@ -3083,6 +3098,11 @@ func (m *Monitor) recordTaskResult(instanceType InstanceType, instance string, p
status.LastErrorAt = time.Time{}
status.LastErrorMessage = ""
status.LastErrorCategory = ""
+ if readErr != nil {
+ status.LastErrorAt = now
+ status.LastErrorMessage = readErr.Error()
+ status.LastErrorCategory = pollReadErrorCategory
+ }
m.mu.Unlock()
if breaker != nil {
breaker.recordSuccess()
diff --git a/internal/monitoring/monitor_additional_test.go b/internal/monitoring/monitor_additional_test.go
index 25058ff5bc..dd78c7cbb1 100644
--- a/internal/monitoring/monitor_additional_test.go
+++ b/internal/monitoring/monitor_additional_test.go
@@ -1882,6 +1882,45 @@ func TestMonitorPersistentFailureBackoffAndRecovery(t *testing.T) {
}
}
+// A downstream read failure after a successful connection is a scheduler
+// success that keeps the read error outstanding in one update (#2960).
+func TestRecordTaskOutcomeReadErrorSchedulesAsSuccess(t *testing.T) {
+ m := &Monitor{
+ config: &config.Config{},
+ circuitBreakers: make(map[string]*circuitBreaker),
+ pollStatusMap: make(map[string]*pollStatus),
+ failureCounts: make(map[string]int),
+ lastOutcome: make(map[string]taskOutcome),
+ }
+ task := ScheduledTask{InstanceType: InstanceTypePMG, InstanceName: "gateway"}
+ key := schedulerKey(task.InstanceType, task.InstanceName)
+ for i := 0; i < 3; i++ {
+ m.recordTaskResult(task.InstanceType, task.InstanceName, errors.New("connection timeout"))
+ }
+ if m.allowExecution(task) {
+ t.Fatal("connection failures did not open the breaker")
+ }
+
+ m.recordTaskOutcome(task.InstanceType, task.InstanceName, nil, errors.New("pmg_queue_status: decode failed"))
+ if outcome := m.lastOutcome[key]; !outcome.success || outcome.err != nil || m.failureCounts[key] != 0 {
+ t.Fatalf("read error scheduled as a failure: outcome=%+v failures=%d", outcome, m.failureCounts[key])
+ }
+ if !m.allowExecution(task) || len(m.SchedulerHealth().Breakers) != 0 {
+ t.Fatal("read error kept the breaker open")
+ }
+ status := m.pollStatusMap[key]
+ if status.ConsecutiveFailures != 0 || status.LastSuccess.IsZero() || !status.FirstFailureAt.IsZero() ||
+ status.LastErrorCategory != pollReadErrorCategory || !strings.Contains(status.LastErrorMessage, "pmg_queue_status") ||
+ !status.LastErrorAt.Equal(status.LastSuccess) {
+ t.Fatalf("read error not kept as the outstanding poll error: %+v", status)
+ }
+
+ m.recordTaskOutcome(task.InstanceType, task.InstanceName, nil, nil)
+ if status.LastErrorMessage != "" || status.LastErrorCategory != "" || !status.LastErrorAt.IsZero() {
+ t.Fatalf("complete poll left the read error outstanding: %+v", status)
+ }
+}
+
func TestGuestConfigPlacement(t *testing.T) {
for _, kind := range []string{"vm", "container", "lxc"} {
for _, reverse := range []bool{false, true} {
diff --git a/internal/monitoring/monitor_pbs_pmg.go b/internal/monitoring/monitor_pbs_pmg.go
index f0f9448bd3..1488592cee 100644
--- a/internal/monitoring/monitor_pbs_pmg.go
+++ b/internal/monitoring/monitor_pbs_pmg.go
@@ -697,7 +697,10 @@ func (m *Monitor) pollPMGInstance(ctx context.Context, instanceName string, clie
start := time.Now()
debugEnabled := logging.IsLevelEnabled(zerolog.DebugLevel)
- var pollErr error
+ // pollErr is a connection, version or whole-poll failure and drives
+ // scheduler backoff. readErr collects downstream reads that failed after
+ // the connection succeeded; see recordTaskOutcome.
+ var pollErr, readErr error
if m.pollMetrics != nil {
m.pollMetrics.IncInFlight("pmg")
defer m.pollMetrics.DecInFlight("pmg")
@@ -722,7 +725,7 @@ func (m *Monitor) pollPMGInstance(ctx context.Context, instanceName string, clie
}()
}
defer func() {
- m.recordTaskResult(InstanceTypePMG, instanceName, pollErr)
+ m.recordTaskOutcome(InstanceTypePMG, instanceName, pollErr, readErr)
}()
select {
@@ -806,10 +809,11 @@ func (m *Monitor) pollPMGInstance(ctx context.Context, instanceName string, clie
m.resetAuthFailures(instanceName, "pmg")
// A successful version read establishes connection health, not complete
- // downstream collection. Preserve missing readings and report failed polls.
+ // downstream collection. Preserve missing readings and report failed reads
+ // without backing off the readings that succeeded.
recordReadError := func(operation string, err error) {
if err != nil {
- pollErr = stderrors.Join(pollErr, errors.WrapAPIError(operation, instanceName, err, 0))
+ readErr = stderrors.Join(readErr, errors.WrapAPIError(operation, instanceName, err, 0))
}
}
@@ -922,7 +926,7 @@ func (m *Monitor) pollPMGInstance(ctx context.Context, instanceName string, clie
// Only a node whose backup list was read is authoritative this cycle. A
// failed cluster read, a failed per-node read or a cancelled poll observed
// nothing, so those nodes keep their last-known backups instead of being
- // published as having none; the failed read is already in pollErr.
+ // published as having none; the failed read is already in readErr.
if clusterErr != nil || len(readBackupNodes) < len(backupNodes) {
var preservedBackupNodes []string
pmgBackups, preservedBackupNodes = preserveUnreadPMGNodeBackups(
@@ -1078,6 +1082,12 @@ func (m *Monitor) pollPMGInstance(ctx context.Context, instanceName string, clie
}
}
+ // Reads cut short by the poll's own deadline or cancellation are a
+ // whole-poll failure, not one broken endpoint, and keep their backoff.
+ if ctxErr := ctx.Err(); ctxErr != nil && readErr != nil {
+ pollErr = stderrors.Join(ctxErr, readErr)
+ }
+
m.state.UpdatePMGBackups(instanceName, pmgBackups)
m.state.UpdatePMGInstance(pmgInst)
log.Info().
diff --git a/internal/monitoring/monitor_pmg_test.go b/internal/monitoring/monitor_pmg_test.go
index 14a7a7070e..96def51631 100644
--- a/internal/monitoring/monitor_pmg_test.go
+++ b/internal/monitoring/monitor_pmg_test.go
@@ -512,16 +512,21 @@ func TestPMGCollectionFailureEvidence(t *testing.T) {
f.fail = op
m.pollPMGInstance(context.Background(), "gateway", f)
status := m.pollStatusMap["pmg::gateway"]
- if status.ConsecutiveFailures != 1 || status.LastErrorMessage == "" || !status.LastSuccess.Equal(before) {
- t.Fatalf("failed read recorded as complete: %+v", status)
- }
v := m.state.GetSnapshot().PMGInstances[0]
if op == "version" {
+ if status.ConsecutiveFailures != 1 || status.LastErrorMessage == "" || !status.LastSuccess.Equal(before) {
+ t.Fatalf("failed connection recorded as complete: %+v", status)
+ }
if v.Status != "offline" || v.ConnectionHealth != "unhealthy" {
t.Fatal("failed connection not offline")
}
return
}
+ // A downstream read failure stays visible on the connection but
+ // is not a scheduler failure (#2960).
+ if status.ConsecutiveFailures != 0 || status.LastErrorMessage == "" || status.LastErrorCategory != pollReadErrorCategory || status.LastSuccess.Before(before) {
+ t.Fatalf("failed read recorded as complete or as a connection failure: %+v", status)
+ }
if v.Status != "online" || v.ConnectionHealth != "healthy" || !m.state.GetSnapshot().ConnectionHealth["pmg-gateway"] {
t.Fatal("downstream failure mislabeled as version connection failure")
}
@@ -714,11 +719,11 @@ func pollPMGBackupRetention(t *testing.T, m *Monitor, f *pmgBackupRetentionFixtu
m.pollPMGInstance(context.Background(), "gateway", f)
}
-func assertPMGPollFailed(t *testing.T, m *Monitor) {
+func assertPMGReadFailed(t *testing.T, m *Monitor) {
t.Helper()
status := m.pollStatusMap["pmg::gateway"]
- if status == nil || status.ConsecutiveFailures == 0 || status.LastErrorMessage == "" {
- t.Fatalf("failed backup enumeration not recorded as a poll failure: %+v", status)
+ if status == nil || status.LastErrorMessage == "" || status.LastErrorCategory != pollReadErrorCategory {
+ t.Fatalf("failed backup enumeration not recorded as a failed read: %+v", status)
}
}
@@ -742,7 +747,7 @@ func TestPMGBackupsSurviveFailedClusterRead(t *testing.T) {
if got := pmgBackupFilesByNode(t, m); !reflect.DeepEqual(got, want) {
t.Fatalf("failed cluster read replaced backups: got=%v want=%v", got, want)
}
- assertPMGPollFailed(t, m)
+ assertPMGReadFailed(t, m)
}
// When every per-node backup read fails, nothing was observed this poll and
@@ -759,7 +764,7 @@ func TestPMGBackupsSurviveEveryNodeBackupReadFailing(t *testing.T) {
if got := pmgBackupFilesByNode(t, m); !reflect.DeepEqual(got, want) {
t.Fatalf("failed backup reads replaced backups: got=%v want=%v", got, want)
}
- assertPMGPollFailed(t, m)
+ assertPMGReadFailed(t, m)
}
// A failed read on one node keeps that node's last-known backups while the
@@ -776,7 +781,7 @@ func TestPMGBackupsKeepFailedNodeOnPartialRead(t *testing.T) {
if got := pmgBackupFilesByNode(t, m); !reflect.DeepEqual(got, want) {
t.Fatalf("partial read backups=%v want=%v", got, want)
}
- assertPMGPollFailed(t, m)
+ assertPMGReadFailed(t, m)
}
// A complete enumeration is authoritative: nodes that answered with no
diff --git a/internal/monitoring/partial_read_scheduling_test.go b/internal/monitoring/partial_read_scheduling_test.go
new file mode 100644
index 0000000000..0eb01e7d2e
--- /dev/null
+++ b/internal/monitoring/partial_read_scheduling_test.go
@@ -0,0 +1,222 @@
+package monitoring
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/config"
+ "github.com/rcourtman/pulse-go-rewrite/pkg/pbs"
+)
+
+// pollAndReschedule runs one poll followed by the scheduler's reschedule step,
+// as the worker loop does, and returns the task the scheduler queued next.
+func pollAndReschedule(t *testing.T, m *Monitor, task ScheduledTask, poll func()) ScheduledTask {
+ t.Helper()
+ poll()
+ m.rescheduleTask(task)
+ next, ok := m.taskQueue.Get(task.InstanceType, task.InstanceName)
+ if !ok {
+ if m.deadLetterQueue.Size() == 0 {
+ t.Fatal("poll was neither rescheduled nor dead-lettered")
+ }
+ return ScheduledTask{}
+ }
+ return next
+}
+
+func newPartialReadPMGMonitor(t *testing.T) (*Monitor, ScheduledTask) {
+ t.Helper()
+ cfg := allPMGScope()
+ cfg.Name = "gateway"
+ m, err := New(&config.Config{PMGInstances: []config.PMGInstance{cfg}})
+ if err != nil {
+ t.Fatalf("failed to create monitor: %v", err)
+ }
+ interval := m.baseIntervalForInstanceType(InstanceTypePMG)
+ if interval <= 0 {
+ t.Fatalf("no PMG base interval: %v", interval)
+ }
+ return m, ScheduledTask{InstanceName: "gateway", InstanceType: InstanceTypePMG, Interval: interval, NextRun: time.Now()}
+}
+
+// Issue #2960: one failed optional PMG read must not back off or dead-letter
+// the connection. The failed reading stays missing and the error stays on the
+// connection, while every other reading keeps its normal cadence.
+func TestPMGOptionalReadFailureKeepsNormalSchedule(t *testing.T) {
+ for _, op := range []string{"cluster", "backup/one", "queue/one", "mail", "count", "scores", "quarantine/spam", "quarantine/virus", "domains", "domain-stats"} {
+ t.Run(op, func(t *testing.T) {
+ m, task := newPartialReadPMGMonitor(t)
+ f := &pmgScopeFixture{calls: map[string]int{}, fail: op}
+
+ for i := 0; i <= m.maxRetryAttempts+1; i++ {
+ next := pollAndReschedule(t, m, task, func() { m.pollPMGInstance(context.Background(), "gateway", f) })
+ if next.Interval != task.Interval {
+ t.Fatalf("poll %d: interval=%v, want normal %v", i, next.Interval, task.Interval)
+ }
+ task = next
+ }
+ if size := m.deadLetterQueue.Size(); size != 0 {
+ t.Fatalf("optional read failure reached the dead-letter queue (%d tasks)", size)
+ }
+ if state, failures, _ := m.ensureBreaker(schedulerKey(InstanceTypePMG, "gateway")).State(); state != "closed" || failures != 0 {
+ t.Fatalf("breaker=%s failures=%d after optional read failures", state, failures)
+ }
+
+ status := m.pollStatusMap["pmg::gateway"]
+ if status.ConsecutiveFailures != 0 || status.LastSuccess.IsZero() {
+ t.Fatalf("partial poll counted as a connection failure: %+v", status)
+ }
+ if status.LastErrorMessage == "" || status.LastErrorCategory != pollReadErrorCategory {
+ t.Fatalf("failed read not left visible on the connection: %+v", status)
+ }
+
+ v := m.state.GetSnapshot().PMGInstances[0]
+ if v.Status != "online" || v.ConnectionHealth != "healthy" {
+ t.Fatalf("partial poll rewrote connection health: %s/%s", v.Status, v.ConnectionHealth)
+ }
+ nodeQueue := func(i int) bool { return len(v.Nodes) > i && v.Nodes[i].QueueStatus != nil }
+ published := map[string]bool{
+ "mail": v.MailStats != nil,
+ "count": len(v.MailCount) > 0,
+ "scores": len(v.SpamDistribution) > 0,
+ "quarantine": v.Quarantine != nil,
+ "domains": len(v.RelayDomains) > 0,
+ "domain-stats": len(v.DomainStats) > 0,
+ "queue/one": nodeQueue(0),
+ "queue/two": nodeQueue(1),
+ }
+ failed := map[string]bool{op: true}
+ switch {
+ case op == "cluster":
+ // Node queues are read per discovered node.
+ failed = map[string]bool{"queue/one": true, "queue/two": true}
+ case strings.HasPrefix(op, "quarantine/"):
+ // Totals require both categories.
+ failed = map[string]bool{"quarantine": true}
+ }
+ for reading, ok := range published {
+ if failed[reading] && ok {
+ t.Errorf("failed %s reading was published", reading)
+ }
+ if !failed[reading] && !ok {
+ t.Errorf("%s reading was not published beside the failed %s read", reading, op)
+ }
+ }
+
+ // A complete poll clears the outstanding read error.
+ f.fail = ""
+ m.pollPMGInstance(context.Background(), "gateway", f)
+ if status.LastErrorMessage != "" || status.LastErrorCategory != "" {
+ t.Fatalf("complete poll left the read error: %+v", status)
+ }
+ })
+ }
+}
+
+// Connection failures keep the scheduler's backoff and dead-letter handling.
+func TestPMGVersionFailureStillBacksOffAndDeadLetters(t *testing.T) {
+ m, task := newPartialReadPMGMonitor(t)
+ f := &pmgScopeFixture{calls: map[string]int{}, fail: "version"}
+
+ next := pollAndReschedule(t, m, task, func() { m.pollPMGInstance(context.Background(), "gateway", f) })
+ if next.Interval == task.Interval {
+ t.Fatalf("version failure kept the normal interval %v", next.Interval)
+ }
+ if status := m.pollStatusMap["pmg::gateway"]; status.ConsecutiveFailures != 1 || status.LastErrorCategory == pollReadErrorCategory {
+ t.Fatalf("version failure not recorded as a connection failure: %+v", status)
+ }
+ for i := 0; i < m.maxRetryAttempts && m.deadLetterQueue.Size() == 0; i++ {
+ pollAndReschedule(t, m, task, func() { m.pollPMGInstance(context.Background(), "gateway", f) })
+ }
+ if m.deadLetterQueue.Size() != 1 {
+ t.Fatal("repeated version failures never reached the dead-letter queue")
+ }
+}
+
+// Reads cut short by the poll's own cancellation are a whole-poll failure,
+// not one broken endpoint, and keep their transient backoff.
+func TestPMGCancelledPollWithFailedReadsBacksOff(t *testing.T) {
+ m, task := newPartialReadPMGMonitor(t)
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ f := &pmgScopeFixture{calls: map[string]int{}, fail: "mail", onVersion: cancel}
+
+ next := pollAndReschedule(t, m, task, func() { m.pollPMGInstance(ctx, "gateway", f) })
+ if next.Interval == task.Interval || m.deadLetterQueue.Size() != 0 {
+ t.Fatalf("cancelled poll interval=%v dlq=%d, want transient backoff", next.Interval, m.deadLetterQueue.Size())
+ }
+ if status := m.pollStatusMap["pmg::gateway"]; status.ConsecutiveFailures != 1 {
+ t.Fatalf("cancelled poll not recorded as a failure: %+v", status)
+ }
+}
+
+// PBS keeps optional read failures (datastores, node status) out of the
+// scheduler outcome already; pin that so a future change cannot reintroduce
+// the PMG coupling behind #2960.
+func TestPBSOptionalReadFailureKeepsNormalSchedule(t *testing.T) {
+ for _, failing := range []string{"/api2/json/admin/datastore", "/api2/json/nodes/localhost/status"} {
+ t.Run(strings.TrimPrefix(failing, "/api2/json/"), func(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ if r.URL.Path == failing {
+ w.WriteHeader(http.StatusInternalServerError)
+ fmt.Fprint(w, `{"message":"fixture read refused","data":null}`)
+ return
+ }
+ switch r.URL.Path {
+ case "/api2/json/version":
+ fmt.Fprint(w, `{"data":{"version":"4.0.1","release":"1","repoid":"fixture"}}`)
+ case "/api2/json/nodes/localhost/status":
+ fmt.Fprint(w, `{"data":{"cpu":0.25,"memory":{"used":512,"total":1024},"uptime":100}}`)
+ case "/api2/json/admin/datastore":
+ fmt.Fprint(w, `{"data":[]}`)
+ default:
+ http.NotFound(w, r)
+ }
+ }))
+ t.Cleanup(server.Close)
+ client, err := pbs.NewClient(pbs.ClientConfig{Host: server.URL, TokenName: "fixture@pbs!reader", TokenValue: "fixture-only"})
+ if err != nil {
+ t.Fatal(err)
+ }
+ m, err := New(&config.Config{PBSInstances: []config.PBSInstance{{
+ Name: "pbs-main", Host: server.URL, TokenName: "fixture@pbs!reader", TokenValue: "fixture-only", MonitorDatastores: true,
+ }}})
+ if err != nil {
+ t.Fatalf("failed to create monitor: %v", err)
+ }
+ task := ScheduledTask{InstanceName: "pbs-main", InstanceType: InstanceTypePBS, Interval: m.baseIntervalForInstanceType(InstanceTypePBS), NextRun: time.Now()}
+
+ for i := 0; i <= m.maxRetryAttempts+1; i++ {
+ next := pollAndReschedule(t, m, task, func() { m.pollPBSInstance(context.Background(), "pbs-main", client) })
+ if next.Interval != task.Interval {
+ t.Fatalf("poll %d: interval=%v, want normal %v", i, next.Interval, task.Interval)
+ }
+ task = next
+ }
+ if size := m.deadLetterQueue.Size(); size != 0 {
+ t.Fatalf("optional read failure reached the dead-letter queue (%d tasks)", size)
+ }
+ if status := m.pollStatusMap["pbs::pbs-main"]; status == nil || status.ConsecutiveFailures != 0 {
+ t.Fatalf("optional read failure counted as a poll failure: %+v", status)
+ }
+
+ inst := pbsInstanceByName(t, m.state.GetSnapshot(), "pbs-main")
+ if inst.Status != "online" || inst.Version != "4.0.1" {
+ t.Fatalf("connection readings not published: %+v", inst)
+ }
+ nodeStatusFailed := failing == "/api2/json/nodes/localhost/status"
+ if inst.NodeMetricsUnavailable != nodeStatusFailed {
+ t.Fatalf("node metrics unavailable=%v with node status failed=%v", inst.NodeMetricsUnavailable, nodeStatusFailed)
+ }
+ if !nodeStatusFailed && inst.CPU != 0.25 {
+ t.Fatalf("node CPU not published beside the failed datastore read: %v", inst.CPU)
+ }
+ })
+ }
+}