Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Preserve capped polling backoff through persistent failures

pulse-triage[bot]59d1183f77d6
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index f1523e8686..8ce332823b 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -17,6 +17,24 @@
## Purpose
+### Sustained poll failures preserve the capped retry fence
+
+Scheduler circuit-breaker backoff saturates before duration arithmetic can
+wrap. Every failure episode retains the existing threshold, exponential
+sequence up to the configured cap and single half-open probe window; prolonged
+failure cannot turn that cap into an immediate retry or a past retry timestamp.
+A successful poll still resets the failure count, interval and outstanding
+error, and one connection's failures do not block another connection.
+
+`TestCircuitBreaker_BackoffNeverOverflows` compares 130 failure transitions
+against an independent unbounded-integer oracle, including odd and maximum
+representable duration caps, exact retry boundaries and success/new-episode
+recovery. `TestMonitorPersistentFailureBackoffAndRecovery` carries persistent
+failure through actual scheduler result/admission and health entry points,
+including unrelated-instance progress and cleared error state. These are
+in-memory source controls, not native guest recovery, appliance request repair
+or installed CPU/network relief. No database or provider operation is involved.
+
### Active-alert state projection carries the live metric status — issue #2068
`activeAlertsSnapshot` copies the alert manager's volatile `MetricStatus`
diff --git a/internal/monitoring/circuit_breaker.go b/internal/monitoring/circuit_breaker.go
index 2ee7e48134..f2dc8547a3 100644
--- a/internal/monitoring/circuit_breaker.go
+++ b/internal/monitoring/circuit_breaker.go
@@ -116,7 +116,17 @@ func (b *circuitBreaker) recordFailure(now time.Time) {
func (b *circuitBreaker) trip(now time.Time) {
b.state = breakerOpen
- delay := b.retryInterval << uint(b.failureCount)
+ // Saturate before doubling. Shifting a time.Duration first can wrap it
+ // negative (or to zero after enough failures), defeating the retry fence
+ // even though the result is subsequently compared with maxDelay.
+ delay := b.retryInterval
+ for remaining := b.failureCount; remaining > 0; remaining-- {
+ if delay > b.maxDelay/2 {
+ delay = b.maxDelay
+ break
+ }
+ delay *= 2
+ }
if delay > b.maxDelay {
delay = b.maxDelay
}
diff --git a/internal/monitoring/circuit_breaker_test.go b/internal/monitoring/circuit_breaker_test.go
index 13efe0fd13..e186e55867 100644
--- a/internal/monitoring/circuit_breaker_test.go
+++ b/internal/monitoring/circuit_breaker_test.go
@@ -1,6 +1,7 @@
package monitoring
import (
+ "math/big"
"testing"
"time"
)
@@ -587,3 +588,71 @@ func TestCircuitBreaker_StateDetails(t *testing.T) {
}
})
}
+
+// Advance the clock rather than waiting or contacting an unhealthy provider.
+// The independent unbounded-integer oracle preserves the existing exponential
+// sequence, while crossing both signed-duration overflow and shift-width bounds.
+func TestCircuitBreaker_BackoffNeverOverflows(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ base, cap time.Duration
+ }{
+ {"default", 5 * time.Second, 5 * time.Minute},
+ {"one minute", 5 * time.Second, time.Minute},
+ {"odd cap", time.Nanosecond, 61 * time.Nanosecond},
+ {"largest duration", time.Nanosecond, time.Duration(1<<63 - 1)},
+ {"large initial duration", time.Duration(1 << 62), time.Duration(1<<63 - 1)},
+ {"base above cap", time.Second, time.Millisecond},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ cb := newCircuitBreaker(3, tc.base, tc.cap, time.Second)
+ now := time.Date(2026, 10, 7, 0, 0, 0, 0, time.UTC)
+ expected := big.NewInt(int64(tc.base))
+ cap := big.NewInt(int64(tc.cap))
+ for failures := 1; failures <= 130; failures++ {
+ cb.recordFailure(now)
+ if failures < 3 {
+ if !cb.allow(now) {
+ t.Fatal("failure threshold changed")
+ }
+ continue
+ }
+ expected.Lsh(expected, uint(failures))
+ if expected.Cmp(cap) > 0 {
+ expected.Set(cap)
+ }
+ state, gotFailures, retryAt := cb.State()
+ want := time.Duration(expected.Int64())
+ if state != "open" || gotFailures != failures || retryAt.Sub(now) != want || cb.retryInterval != want {
+ t.Fatalf("failure %d: state=%s count=%d delay=%v retry=%v, want %v", failures, state, gotFailures, cb.retryInterval, retryAt.Sub(now), want)
+ }
+ if cb.allow(retryAt.Add(-time.Nanosecond)) {
+ t.Fatalf("failure %d admitted an early retry", failures)
+ }
+ if !cb.allow(retryAt) {
+ t.Fatalf("failure %d denied the due probe", failures)
+ }
+ if cb.allow(retryAt) {
+ t.Fatalf("failure %d admitted a second half-open probe", failures)
+ }
+ now = retryAt
+ }
+ cb.recordSuccess()
+ state, failures, retryAt := cb.State()
+ if state != "closed" || failures != 0 || !retryAt.IsZero() || cb.retryInterval != tc.base || !cb.allow(now) {
+ t.Fatal("successful probe did not restore the original healthy admission")
+ }
+ for i := 0; i < 3; i++ {
+ cb.recordFailure(now)
+ }
+ expected.SetInt64(int64(tc.base))
+ expected.Lsh(expected, 3)
+ if expected.Cmp(cap) > 0 {
+ expected.Set(cap)
+ }
+ if cb.retryInterval != time.Duration(expected.Int64()) {
+ t.Fatal("a new failure episode retained the old saturated delay")
+ }
+ })
+ }
+}
diff --git a/internal/monitoring/monitor_additional_test.go b/internal/monitoring/monitor_additional_test.go
index f854346ad2..399421f227 100644
--- a/internal/monitoring/monitor_additional_test.go
+++ b/internal/monitoring/monitor_additional_test.go
@@ -2,6 +2,7 @@ package monitoring
import (
"context"
+ "errors"
"net"
"testing"
"time"
@@ -1795,3 +1796,54 @@ func TestNodeTemperatureStaysWithinSameNamedNodesSite(t *testing.T) {
}
})
}
+
+// Exercise the actual scheduler result, admission and health entry points.
+// No provider request, backup operation or database is involved in this control.
+func TestMonitorPersistentFailureBackoffAndRecovery(t *testing.T) {
+ m := &Monitor{
+ config: &config.Config{},
+ circuitBreakers: make(map[string]*circuitBreaker),
+ pollStatusMap: make(map[string]*pollStatus),
+ failureCounts: make(map[string]int),
+ lastOutcome: make(map[string]taskOutcome),
+ }
+ failed := ScheduledTask{InstanceType: InstanceTypePVE, InstanceName: "offline"}
+ healthy := ScheduledTask{InstanceType: InstanceTypePBS, InstanceName: "healthy"}
+ for failure := 1; failure <= 130; failure++ {
+ m.recordTaskResult(failed.InstanceType, failed.InstanceName, errors.New("connection timeout"))
+ if failure < 3 {
+ continue
+ }
+ cb := m.ensureBreaker(schedulerKey(failed.InstanceType, failed.InstanceName))
+ state, count, retryAt := cb.State()
+ if state != "open" || count != failure || !retryAt.After(time.Now()) || m.allowExecution(failed) {
+ t.Fatalf("failure %d lost scheduler containment: state=%s count=%d retry=%v", failure, state, count, retryAt)
+ }
+ if !m.allowExecution(healthy) {
+ t.Fatal("one failing connection blocked another instance")
+ }
+ health := m.SchedulerHealth()
+ if len(health.Breakers) != 1 || health.Breakers[0].Failures != failure || !health.Breakers[0].RetryAt.Equal(retryAt) || health.Breakers[0].State != "open" {
+ t.Fatalf("served scheduler health lost the owned retry fence: %+v", health.Breakers)
+ }
+ // Simulate reaching the scheduled retry without sleeping or polling a host.
+ if !cb.allow(retryAt) {
+ t.Fatal("due probe was not admitted")
+ }
+ }
+ m.recordTaskResult(failed.InstanceType, failed.InstanceName, nil)
+ if !m.allowExecution(failed) || len(m.SchedulerHealth().Breakers) != 0 {
+ t.Fatal("successful poll did not restore scheduler admission and health")
+ }
+ status := m.pollStatusMap[schedulerKey(failed.InstanceType, failed.InstanceName)]
+ if status.ConsecutiveFailures != 0 || !status.LastErrorAt.IsZero() || status.LastErrorMessage != "" || status.LastErrorCategory != "" || status.LastSuccess.IsZero() {
+ t.Fatalf("successful poll retained an outstanding failure: %+v", status)
+ }
+ for i := 0; i < 3; i++ {
+ m.recordTaskResult(failed.InstanceType, failed.InstanceName, errors.New("connection timeout"))
+ }
+ cb := m.ensureBreaker(schedulerKey(failed.InstanceType, failed.InstanceName))
+ if cb.retryInterval != 40*time.Second || m.allowExecution(failed) {
+ t.Fatal("new failure episode did not restart the original backoff")
+ }
+}