Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

2.6 KB
Raw Blame
package server

import (
	"runtime"
	"sync"
	"time"

	"github.com/rs/zerolog"
)

// startupWatchdogBudget bounds how long Run may spend between binding the
// UI/API listener and actually serving it.
//
// The listener is bound early so a port conflict fails fast, but nothing
// accepts connections until srv.Serve starts. Every synchronous initialization
// step in between (monitor start, router construction, optional AI/relay/
// telemetry/config-watcher startup) therefore runs with a live listening
// socket whose accept queue is filling and no reader. A normal startup
// completes in seconds; this budget only trips when one of those steps stalls,
// which is otherwise invisible because no log line is emitted while it runs.
const startupWatchdogBudget = 2 * time.Minute

// startupWatchdogStackBytes is the buffer reserved for the goroutine dump. The
// default 64KiB used elsewhere truncates under a large goroutine set; a stalled
// startup is exactly when the full stack matters.
const startupWatchdogStackBytes = 256 * 1024

// startupPhase records the last completed initialization step so a watchdog
// dump names where Run was when it stalled instead of only showing the stacks.
type startupPhase struct {
	mu    sync.Mutex
	phase string
}

func (s *startupPhase) mark(phase string) {
	if s == nil {
		return
	}
	s.mu.Lock()
	s.phase = phase
	s.mu.Unlock()
}

func (s *startupPhase) current() string {
	if s == nil {
		return "unknown"
	}
	s.mu.Lock()
	defer s.mu.Unlock()
	if s.phase == "" {
		return "unknown"
	}
	return s.phase
}

// startStartupWatchdog arms an independent timer that fires only if the server
// has not started serving within budget. It logs the last completed startup
// phase and every goroutine stack so a recurrence of the reported
// bound-but-not-serving startup stall (issue #2129) is diagnosable from the
// field log alone, without waiting for an operator to send SIGQUIT. The
// returned stop function must be called once serving begins or Run exits; it is
// safe to call more than once.
func startStartupWatchdog(logger zerolog.Logger, phase *startupPhase, budget time.Duration) (stop func()) {
	done := make(chan struct{})
	var once sync.Once
	go func() {
		timer := time.NewTimer(budget)
		defer timer.Stop()
		select {
		case <-done:
		case <-timer.C:
			buf := make([]byte, startupWatchdogStackBytes)
			n := runtime.Stack(buf, true)
			logger.Error().
				Dur("budget", budget).
				Str("last_phase", phase.current()).
				Str("goroutines", string(buf[:n])).
				Msg("Pulse has not started serving its UI/API listener within the startup watchdog budget; startup is stalled")
		}
	}()
	return func() {
		once.Do(func() { close(done) })
	}
}