Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Preserve capped polling backoff through persistent failures
pulse-triage[bot]59d1183f77d6
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex f1523e8686..8ce332823b 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -17,6 +17,24 @@## Purpose+### Sustained poll failures preserve the capped retry fence++Scheduler circuit-breaker backoff saturates before duration arithmetic can+wrap. Every failure episode retains the existing threshold, exponential+sequence up to the configured cap and single half-open probe window; prolonged+failure cannot turn that cap into an immediate retry or a past retry timestamp.+A successful poll still resets the failure count, interval and outstanding+error, and one connection's failures do not block another connection.++`TestCircuitBreaker_BackoffNeverOverflows` compares 130 failure transitions+against an independent unbounded-integer oracle, including odd and maximum+representable duration caps, exact retry boundaries and success/new-episode+recovery. `TestMonitorPersistentFailureBackoffAndRecovery` carries persistent+failure through actual scheduler result/admission and health entry points,+including unrelated-instance progress and cleared error state. These are+in-memory source controls, not native guest recovery, appliance request repair+or installed CPU/network relief. No database or provider operation is involved.+### Active-alert state projection carries the live metric status — issue #2068`activeAlertsSnapshot` copies the alert manager's volatile `MetricStatus`diff --git a/internal/monitoring/circuit_breaker.go b/internal/monitoring/circuit_breaker.goindex 2ee7e48134..f2dc8547a3 100644--- a/internal/monitoring/circuit_breaker.go+++ b/internal/monitoring/circuit_breaker.go@@ -116,7 +116,17 @@ func (b *circuitBreaker) recordFailure(now time.Time) {func (b *circuitBreaker) trip(now time.Time) {b.state = breakerOpen- delay := b.retryInterval << uint(b.failureCount)+ // Saturate before doubling. Shifting a time.Duration first can wrap it+ // negative (or to zero after enough failures), defeating the retry fence+ // even though the result is subsequently compared with maxDelay.+ delay := b.retryInterval+ for remaining := b.failureCount; remaining > 0; remaining-- {+ if delay > b.maxDelay/2 {+ delay = b.maxDelay+ break+ }+ delay *= 2+ }if delay > b.maxDelay {delay = b.maxDelay}diff --git a/internal/monitoring/circuit_breaker_test.go b/internal/monitoring/circuit_breaker_test.goindex 13efe0fd13..e186e55867 100644--- a/internal/monitoring/circuit_breaker_test.go+++ b/internal/monitoring/circuit_breaker_test.go@@ -1,6 +1,7 @@package monitoringimport (+ "math/big""testing""time")@@ -587,3 +588,71 @@ func TestCircuitBreaker_StateDetails(t *testing.T) {}})}++// Advance the clock rather than waiting or contacting an unhealthy provider.+// The independent unbounded-integer oracle preserves the existing exponential+// sequence, while crossing both signed-duration overflow and shift-width bounds.+func TestCircuitBreaker_BackoffNeverOverflows(t *testing.T) {+ for _, tc := range []struct {+ name string+ base, cap time.Duration+ }{+ {"default", 5 * time.Second, 5 * time.Minute},+ {"one minute", 5 * time.Second, time.Minute},+ {"odd cap", time.Nanosecond, 61 * time.Nanosecond},+ {"largest duration", time.Nanosecond, time.Duration(1<<63 - 1)},+ {"large initial duration", time.Duration(1 << 62), time.Duration(1<<63 - 1)},+ {"base above cap", time.Second, time.Millisecond},+ } {+ t.Run(tc.name, func(t *testing.T) {+ cb := newCircuitBreaker(3, tc.base, tc.cap, time.Second)+ now := time.Date(2026, 10, 7, 0, 0, 0, 0, time.UTC)+ expected := big.NewInt(int64(tc.base))+ cap := big.NewInt(int64(tc.cap))+ for failures := 1; failures <= 130; failures++ {+ cb.recordFailure(now)+ if failures < 3 {+ if !cb.allow(now) {+ t.Fatal("failure threshold changed")+ }+ continue+ }+ expected.Lsh(expected, uint(failures))+ if expected.Cmp(cap) > 0 {+ expected.Set(cap)+ }+ state, gotFailures, retryAt := cb.State()+ want := time.Duration(expected.Int64())+ if state != "open" || gotFailures != failures || retryAt.Sub(now) != want || cb.retryInterval != want {+ t.Fatalf("failure %d: state=%s count=%d delay=%v retry=%v, want %v", failures, state, gotFailures, cb.retryInterval, retryAt.Sub(now), want)+ }+ if cb.allow(retryAt.Add(-time.Nanosecond)) {+ t.Fatalf("failure %d admitted an early retry", failures)+ }+ if !cb.allow(retryAt) {+ t.Fatalf("failure %d denied the due probe", failures)+ }+ if cb.allow(retryAt) {+ t.Fatalf("failure %d admitted a second half-open probe", failures)+ }+ now = retryAt+ }+ cb.recordSuccess()+ state, failures, retryAt := cb.State()+ if state != "closed" || failures != 0 || !retryAt.IsZero() || cb.retryInterval != tc.base || !cb.allow(now) {+ t.Fatal("successful probe did not restore the original healthy admission")+ }+ for i := 0; i < 3; i++ {+ cb.recordFailure(now)+ }+ expected.SetInt64(int64(tc.base))+ expected.Lsh(expected, 3)+ if expected.Cmp(cap) > 0 {+ expected.Set(cap)+ }+ if cb.retryInterval != time.Duration(expected.Int64()) {+ t.Fatal("a new failure episode retained the old saturated delay")+ }+ })+ }+}diff --git a/internal/monitoring/monitor_additional_test.go b/internal/monitoring/monitor_additional_test.goindex f854346ad2..399421f227 100644--- a/internal/monitoring/monitor_additional_test.go+++ b/internal/monitoring/monitor_additional_test.go@@ -2,6 +2,7 @@ package monitoringimport ("context"+ "errors""net""testing""time"@@ -1795,3 +1796,54 @@ func TestNodeTemperatureStaysWithinSameNamedNodesSite(t *testing.T) {}})}++// Exercise the actual scheduler result, admission and health entry points.+// No provider request, backup operation or database is involved in this control.+func TestMonitorPersistentFailureBackoffAndRecovery(t *testing.T) {+ m := &Monitor{+ config: &config.Config{},+ circuitBreakers: make(map[string]*circuitBreaker),+ pollStatusMap: make(map[string]*pollStatus),+ failureCounts: make(map[string]int),+ lastOutcome: make(map[string]taskOutcome),+ }+ failed := ScheduledTask{InstanceType: InstanceTypePVE, InstanceName: "offline"}+ healthy := ScheduledTask{InstanceType: InstanceTypePBS, InstanceName: "healthy"}+ for failure := 1; failure <= 130; failure++ {+ m.recordTaskResult(failed.InstanceType, failed.InstanceName, errors.New("connection timeout"))+ if failure < 3 {+ continue+ }+ cb := m.ensureBreaker(schedulerKey(failed.InstanceType, failed.InstanceName))+ state, count, retryAt := cb.State()+ if state != "open" || count != failure || !retryAt.After(time.Now()) || m.allowExecution(failed) {+ t.Fatalf("failure %d lost scheduler containment: state=%s count=%d retry=%v", failure, state, count, retryAt)+ }+ if !m.allowExecution(healthy) {+ t.Fatal("one failing connection blocked another instance")+ }+ health := m.SchedulerHealth()+ if len(health.Breakers) != 1 || health.Breakers[0].Failures != failure || !health.Breakers[0].RetryAt.Equal(retryAt) || health.Breakers[0].State != "open" {+ t.Fatalf("served scheduler health lost the owned retry fence: %+v", health.Breakers)+ }+ // Simulate reaching the scheduled retry without sleeping or polling a host.+ if !cb.allow(retryAt) {+ t.Fatal("due probe was not admitted")+ }+ }+ m.recordTaskResult(failed.InstanceType, failed.InstanceName, nil)+ if !m.allowExecution(failed) || len(m.SchedulerHealth().Breakers) != 0 {+ t.Fatal("successful poll did not restore scheduler admission and health")+ }+ status := m.pollStatusMap[schedulerKey(failed.InstanceType, failed.InstanceName)]+ if status.ConsecutiveFailures != 0 || !status.LastErrorAt.IsZero() || status.LastErrorMessage != "" || status.LastErrorCategory != "" || status.LastSuccess.IsZero() {+ t.Fatalf("successful poll retained an outstanding failure: %+v", status)+ }+ for i := 0; i < 3; i++ {+ m.recordTaskResult(failed.InstanceType, failed.InstanceName, errors.New("connection timeout"))+ }+ cb := m.ensureBreaker(schedulerKey(failed.InstanceType, failed.InstanceName))+ if cb.retryInterval != 40*time.Second || m.allowExecution(failed) {+ t.Fatal("new failure episode did not restart the original backoff")+ }+}