Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Keep missing startup connections eligible for ordinary recovery

pulse-triage[bot]1080e2f9ddd9
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index 1bdea46412..b0dccf390d 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -4263,6 +4263,23 @@ and read-state preservation.
### PBS health is one completed-poll outcome
+Configured, active PVE/PBS connections whose initial client could not be
+constructed remain eligible for startup recovery for the monitor's lifetime.
+The existing 5/10/20/40/60-second backoff stays capped at sixty seconds; a
+five-minute startup window must not permanently strand a missing client
+outside the providers' ordinary polling lists. Cancellation interrupts the
+wait, paused instances are excluded, and constructor retries preserve the
+configured authority, authentication and TLS trust. This does not add a PMG
+retry route or change existing clients' request/renewal policies.
+`TestConnectionRetrySurvivesLongStartupOutage` advances the real timers with
+Go's virtual clock through a six-minute outage and checks every attempt's
+cadence and unchanged connection inputs. The cancellation and pause controls
+pin lifecycle/exclusion behaviour. `TestPBSStartupRecoveryKeepsTrustAndFirstPollBoundary`
+exercises the real pinned-TLS PBS constructor, ordinary scheduler task and
+broadcast producer: a recreated client still has no successful poll evidence,
+an accepted first poll restores measured host/datastore values, and a wrong
+saved pin remains refused without password-bearing HTTP or pin replacement.
+
PBS client construction is transport setup, not connectivity evidence. Initial
client creation and retry recreation therefore remain pending until a poll
completes; client-construction failures publish a failed scheduler result
diff --git a/internal/monitoring/canonical_guardrails_test.go b/internal/monitoring/canonical_guardrails_test.go
index 748d6043b9..c8ed1c7438 100644
--- a/internal/monitoring/canonical_guardrails_test.go
+++ b/internal/monitoring/canonical_guardrails_test.go
@@ -118,6 +118,15 @@ func TestPBSClientConstructionCannotPublishConnectedHealth(t *testing.T) {
}
}
+// Keep the registered monitoring proof connected to executable lifecycle,
+// trust and actual producer controls, not a source-text expiry assertion.
+func TestStartupConnectionRecoveryCanonicalContract(t *testing.T) {
+ t.Run("late-recovery", TestConnectionRetrySurvivesLongStartupOutage)
+ t.Run("cancel-capped-wait", TestConnectionRetryLongOutageStopsOnCancellation)
+ t.Run("paused-exclusions", TestConnectionRetryDoesNotResumePausedConnections)
+ t.Run("pbs-trust-and-first-poll", TestPBSStartupRecoveryKeepsTrustAndFirstPollBoundary)
+}
+
func TestMultiTenantMonitorListOrganizationIDsFallsBackToDefault(t *testing.T) {
got, err := (&MultiTenantMonitor{}).ListOrganizationIDs()
if err != nil {
diff --git a/internal/monitoring/monitor_client_reconnect.go b/internal/monitoring/monitor_client_reconnect.go
index f261258f0b..eb0296137c 100644
--- a/internal/monitoring/monitor_client_reconnect.go
+++ b/internal/monitoring/monitor_client_reconnect.go
@@ -21,30 +21,25 @@ var connRetryDelays = []time.Duration{
}
// retryFailedConnections attempts to recreate clients that failed during initialization
-// This handles cases where Proxmox/network isn't ready when Pulse starts
+// This handles cases where Proxmox/network isn't ready when Pulse starts. Keep
+// the capped retry cadence until every active connection has a client or the
+// monitor stops: providers list clients, so an expired startup retry would leave
+// a missing connection outside ordinary polling permanently.
func (m *Monitor) retryFailedConnections(ctx context.Context) {
defer recoverFromPanic("retryFailedConnections")
- // Retry schedule: 5s, 10s, 20s, 40s, 60s, then every 60s for up to 5 minutes total
+ // Retry schedule: 5s, 10s, 20s, 40s, 60s, then every 60s.
retryDelays := connRetryDelays
-
- maxRetryDuration := 5 * time.Minute
- startTime := time.Now()
retryIndex := 0
for {
- // Stop retrying after max duration or if context is cancelled
+ // Stop when the monitor is cancelled, including while waiting below.
select {
case <-ctx.Done():
return
default:
}
- if time.Since(startTime) > maxRetryDuration {
- log.Info().Msg("connection retry period expired")
- return
- }
-
// Calculate next retry delay
var delay time.Duration
if retryIndex < len(retryDelays) {
diff --git a/internal/monitoring/monitor_startup_recovery_test.go b/internal/monitoring/monitor_startup_recovery_test.go
new file mode 100644
index 0000000000..39287d3bf9
--- /dev/null
+++ b/internal/monitoring/monitor_startup_recovery_test.go
@@ -0,0 +1,192 @@
+package monitoring
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "reflect"
+ "strings"
+ "sync/atomic"
+ "testing"
+ "testing/synctest"
+ "time"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/config"
+ "github.com/rcourtman/pulse-go-rewrite/internal/models"
+ "github.com/rcourtman/pulse-go-rewrite/pkg/pbs"
+ "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
+)
+
+// Advance the real retry timers with Go's virtual clock, not shorter delays or
+// a changed cutoff. A provider returning after six minutes must become eligible
+// for ordinary polling on the same running monitor, with unchanged TLS inputs.
+func TestConnectionRetrySurvivesLongStartupOutage(t *testing.T) {
+ original := newProxmoxClientFunc
+ defer func() { newProxmoxClientFunc = original }()
+ synctest.Test(t, func(t *testing.T) {
+ instance := config.PVEInstance{Name: "late-pve", Host: "https://pve.local:8006", User: "fixture@pam", Password: "fixture-password", VerifySSL: true, Fingerprint: "fixture-saved-pin"}
+ m := &Monitor{config: &config.Config{PVEInstances: []config.PVEInstance{instance}, ConnectionTimeout: 3 * time.Second}, pveClients: make(map[string]PVEClientInterface), pbsClients: make(map[string]*pbs.Client), state: models.NewState()}
+ started := time.Now()
+ var attempts []time.Duration
+ newProxmoxClientFunc = func(cfg proxmox.ClientConfig) (PVEClientInterface, error) {
+ want := config.CreateProxmoxConfig(&instance)
+ want.Timeout = 3 * time.Second
+ if !reflect.DeepEqual(cfg, want) {
+ t.Fatal("retry changed configured authority, credentials, TLS pin or timeout")
+ }
+ elapsed := time.Since(started)
+ attempts = append(attempts, elapsed)
+ if elapsed < 6*time.Minute {
+ return nil, fmt.Errorf("fixture provider still starting")
+ }
+ return &mockPVEClientExtra{}, nil
+ }
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
+ defer cancel()
+ m.retryFailedConnections(ctx)
+ client, exists := m.getPVEClient(instance.Name)
+ if !exists || client == nil || len(newPVEPollProvider().ListInstances(m)) != 1 {
+ t.Fatalf("six-minute startup outage left the connection permanently outside polling; attempts=%v", attempts)
+ }
+ want := []time.Duration{5 * time.Second, 15 * time.Second, 35 * time.Second, 75 * time.Second, 135 * time.Second, 195 * time.Second, 255 * time.Second, 315 * time.Second, 375 * time.Second}
+ if !reflect.DeepEqual(attempts, want) {
+ t.Fatalf("retry cadence=%v, want capped schedule=%v", attempts, want)
+ }
+ t.Logf("late startup recovered after %s, attempts=%v; ordinary provider lists the retained client", attempts[len(attempts)-1], attempts)
+ })
+}
+
+func TestConnectionRetryLongOutageStopsOnCancellation(t *testing.T) {
+ original := newProxmoxClientFunc
+ defer func() { newProxmoxClientFunc = original }()
+ synctest.Test(t, func(t *testing.T) {
+ m := &Monitor{config: &config.Config{PVEInstances: []config.PVEInstance{{Name: "still-unavailable", Host: "https://pve.local:8006"}}}, pveClients: make(map[string]PVEClientInterface), pbsClients: make(map[string]*pbs.Client), state: models.NewState()}
+ calls := 0
+ newProxmoxClientFunc = func(proxmox.ClientConfig) (PVEClientInterface, error) {
+ calls++
+ return nil, fmt.Errorf("fixture provider still unavailable")
+ }
+ started := time.Now()
+ ctx, cancel := context.WithTimeout(context.Background(), 6*time.Minute+10*time.Second)
+ defer cancel()
+ m.retryFailedConnections(ctx)
+ if elapsed := time.Since(started); elapsed != 6*time.Minute+10*time.Second || calls != 8 || len(m.pveClients) != 0 {
+ t.Fatalf("shutdown must interrupt the capped wait without another attempt: elapsed=%s calls=%d clients=%d", elapsed, calls, len(m.pveClients))
+ }
+ })
+}
+
+func TestConnectionRetryDoesNotResumePausedConnections(t *testing.T) {
+ original := newProxmoxClientFunc
+ defer func() { newProxmoxClientFunc = original }()
+ synctest.Test(t, func(t *testing.T) {
+ m := &Monitor{config: &config.Config{PVEInstances: []config.PVEInstance{{Name: "paused-pve", Disabled: true}}, PBSInstances: []config.PBSInstance{{Name: "paused-pbs", Disabled: true, Host: "://invalid"}}}, pveClients: make(map[string]PVEClientInterface), pbsClients: make(map[string]*pbs.Client), state: models.NewState()}
+ newProxmoxClientFunc = func(proxmox.ClientConfig) (PVEClientInterface, error) {
+ t.Fatal("retry attempted a paused connection")
+ return nil, nil
+ }
+ m.retryFailedConnections(context.Background())
+ if len(m.pveClients) != 0 || len(m.pbsClients) != 0 || len(m.state.GetSnapshot().ConnectionHealth) != 0 {
+ t.Fatal("paused connections were recreated or published as healthy")
+ }
+ })
+}
+
+// The clock test above exercises the production six-minute/capped schedule.
+// This independent, closed TLS fixture exercises PBS's actual constructor,
+// scheduler admission, first poll and broadcast producer. Only the waits are
+// shortened here; no credentials, trust, daemon, backup or guest operation runs.
+func TestPBSStartupRecoveryKeepsTrustAndFirstPollBoundary(t *testing.T) {
+ var logins atomic.Int32
+ server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ if r.URL.Path == "/api2/json/access/ticket" {
+ if logins.Add(1) <= 2 {
+ w.WriteHeader(http.StatusServiceUnavailable)
+ fmt.Fprint(w, `{"message":"fixture starting"}`)
+ return
+ }
+ fmt.Fprint(w, `{"data":{"ticket":"fixture-ticket","CSRFPreventionToken":"fixture-csrf"}}`)
+ return
+ }
+ if r.Method != http.MethodGet || !strings.Contains(r.Header.Get("Cookie"), "fixture-ticket") {
+ t.Errorf("ordinary PBS read lost authentication or used a mutation: %s %s", r.Method, r.URL.Path)
+ http.Error(w, "fixture refuses unauthenticated reads", http.StatusForbidden)
+ return
+ }
+ switch r.URL.Path {
+ case "/api2/json/version":
+ fmt.Fprint(w, `{"data":{"version":"3.4.2"}}`)
+ case "/api2/json/nodes/localhost/status":
+ fmt.Fprint(w, `{"data":{"cpu":0.15,"memory":{"used":512,"total":1024},"uptime":120}}`)
+ case "/api2/json/admin/datastore":
+ fmt.Fprint(w, `{"data":[{"store":"backups","total":1000,"used":250,"avail":750}]}`)
+ case "/api2/json/admin/datastore/backups/status":
+ fmt.Fprint(w, `{"data":{"status":"available","total":1000,"used":250,"avail":750}}`)
+ default:
+ fmt.Fprint(w, `{"data":[]}`)
+ }
+ }))
+ defer server.Close()
+ pin := sha256.Sum256(server.Certificate().Raw)
+ instance := config.PBSInstance{Name: "startup-pbs", Host: server.URL, User: "fixture@pbs", Password: "fixture-password", VerifySSL: true, Fingerprint: hex.EncodeToString(pin[:]), MonitorDatastores: true}
+ m := newPBSHealthAuthorityMonitor([]config.PBSInstance{instance})
+ project := func(phase string) {
+ wire, err := json.Marshal(struct {
+ Phase string `json:"phase"`
+ State models.StateFrontend `json:"state"`
+ }{phase, m.buildBroadcastFrontendStateFromSnapshot(m.state.GetSnapshot(), mockModeScope{})})
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Logf("PBS_STARTUP_PROJECTION_JSON=%s", wire)
+ }
+ m.initPBSClients(m.config)
+ assertPBSConnectionProjection(t, m, instance.Name, false, "offline")
+ if len(newPBSPollProvider().ListInstances(m)) != 0 {
+ t.Fatal("failed constructor entered polling")
+ }
+ project("unavailable")
+ originalDelays := connRetryDelays
+ connRetryDelays = []time.Duration{time.Millisecond, time.Millisecond, time.Millisecond}
+ defer func() { connRetryDelays = originalDelays }()
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ m.retryFailedConnections(ctx)
+ client, exists := m.getPBSClient(instance.Name)
+ if !exists || client == nil || logins.Load() != 3 || len(newPBSPollProvider().ListInstances(m)) != 1 {
+ t.Fatalf("ordinary PBS startup recovery missing or duplicated: exists=%t logins=%d", exists, logins.Load())
+ }
+ assertPBSConnectionProjection(t, m, instance.Name, false, "offline")
+ project("awaiting-poll")
+ task, err := newPBSPollProvider().BuildPollTask(m, instance.Name)
+ if err != nil {
+ t.Fatal(err)
+ }
+ newRealExecutor(m).Execute(ctx, task)
+ assertPBSConnectionProjection(t, m, instance.Name, true, "online")
+ published := pbsInstanceByName(t, m.state.GetSnapshot(), instance.Name)
+ if published.CPU != 15 || published.Memory != 50 || len(published.Datastores) != 1 || published.Datastores[0].Usage != 25 || logins.Load() != 3 || !reflect.DeepEqual(instance, m.config.PBSInstances[0]) {
+ t.Fatalf("recovery lost native readings/identity/trust or replayed authentication: %+v logins=%d", published, logins.Load())
+ }
+ project("recovered")
+
+ // A different saved pin is still a refusal, not recovery authority. The
+ // same real retry path must not make a password-bearing HTTP request.
+ wrong := instance
+ wrong.Name = "wrong-pin-pbs"
+ wrong.Fingerprint = strings.Repeat("0", 64)
+ untrusted := newPBSHealthAuthorityMonitor([]config.PBSInstance{wrong})
+ untrusted.initPBSClients(untrusted.config)
+ refusedCtx, refusedCancel := context.WithTimeout(context.Background(), 25*time.Millisecond)
+ defer refusedCancel()
+ untrusted.retryFailedConnections(refusedCtx)
+ if len(untrusted.pbsClients) != 0 || logins.Load() != 3 || untrusted.config.PBSInstances[0].Fingerprint != wrong.Fingerprint {
+ t.Fatal("retry bypassed or replaced a saved TLS pin")
+ }
+ assertPBSConnectionProjection(t, untrusted, wrong.Name, false, "offline")
+}