Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package server
import (
"context"
"crypto/tls"
"encoding/json"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/url"
"regexp"
"strings"
"syscall"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/telemetry"
)
const (
serviceHealthProbeTimeout = 5 * time.Second
serviceHealthRetryDelay = time.Second
serviceHealthBodyLimit = 2 << 20
serviceHealthAssetLimit = 32
)
var frontendAssetReferencePattern = regexp.MustCompile(`(?i)(?:src|href)\s*=\s*["'](/assets/[^"'#?]+(?:\?[^"'#]*)?)["']`)
func newServiceHealthProbe(listener net.Listener, tlsEnabled bool) func() telemetry.ServiceHealthObservation {
baseURLs := localServiceHealthBaseURLs(listener, tlsEnabled)
if len(baseURLs) == 0 {
return func() telemetry.ServiceHealthObservation {
return unhealthyServiceObservation(telemetry.ServiceHealthFailureAPIConnectivity)
}
}
transport := &http.Transport{}
if tlsEnabled {
// The probe connects only to the address already bound by listener
// (or loopback for a wildcard). Certificate trust is a client-facing
// concern; this verifies Pulse can serve its own HTTPS handler.
transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} // #nosec G402
}
client := &http.Client{
Transport: transport,
Timeout: serviceHealthProbeTimeout,
CheckRedirect: func(_ *http.Request, _ []*http.Request) error {
return http.ErrUseLastResponse
},
}
return serviceHealthProbe(baseURLs, client, serviceHealthProbeTimeout, serviceHealthRetryDelay)
}
// Each observation gets at most two five-second attempts with one second for
// startup to settle between them. Telemetry invokes this in its background
// runner, never on the monitoring or HTTP serving path.
func serviceHealthProbe(baseURLs []string, client *http.Client, timeout, retryDelay time.Duration) func() telemetry.ServiceHealthObservation {
return func() telemetry.ServiceHealthObservation {
defer client.CloseIdleConnections()
var observation telemetry.ServiceHealthObservation
for attempt := 0; attempt < 2; attempt++ {
if attempt > 0 {
timer := time.NewTimer(retryDelay)
<-timer.C
}
ctx, cancel := context.WithTimeout(context.Background(), timeout)
observation = observeServiceHealth(ctx, client, baseURLs)
cancel()
if observation.Healthy {
return observation
}
}
return observation
}
}
func observeServiceHealth(ctx context.Context, client *http.Client, baseURLs []string) telemetry.ServiceHealthObservation {
baseURL, apiBody, status, err := serviceHealthAPI(ctx, client, baseURLs)
if err != nil {
category := telemetry.ServiceHealthFailureAPIConnectivity
if status != 0 {
category = telemetry.ServiceHealthFailureAPIStatus
}
return unhealthyServiceObservation(serviceHealthErrorCategory(err, category))
}
if status < http.StatusOK || status >= http.StatusMultipleChoices {
return unhealthyServiceObservation(telemetry.ServiceHealthFailureAPIStatus)
}
var health struct {
Status string `json:"status"`
}
if json.Unmarshal(apiBody, &health) != nil || health.Status != "healthy" {
return unhealthyServiceObservation(telemetry.ServiceHealthFailureAPIStatus)
}
indexBody, status, err := serviceHealthGET(ctx, client, baseURL+"/")
if err != nil {
return unhealthyServiceObservation(serviceHealthErrorCategory(err, telemetry.ServiceHealthFailureUIStatus))
}
if status < http.StatusOK || status >= http.StatusMultipleChoices ||
!strings.Contains(strings.ToLower(string(indexBody)), "<html") {
return unhealthyServiceObservation(telemetry.ServiceHealthFailureUIStatus)
}
assetPaths := frontendAssetPaths(indexBody)
if len(assetPaths) == 0 {
return unhealthyServiceObservation(telemetry.ServiceHealthFailureFrontendAssets)
}
for _, assetPath := range assetPaths {
body, assetStatus, assetErr := serviceHealthGET(ctx, client, baseURL+assetPath)
if assetErr != nil {
return unhealthyServiceObservation(serviceHealthErrorCategory(assetErr, telemetry.ServiceHealthFailureFrontendAssets))
}
if assetStatus < http.StatusOK || assetStatus >= http.StatusMultipleChoices || len(body) == 0 {
return unhealthyServiceObservation(telemetry.ServiceHealthFailureFrontendAssets)
}
}
return telemetry.ServiceHealthObservation{Observed: true, Healthy: true}
}
// Only a transport failure tries the other loopback family. A response pins
// the entire API/UI/asset observation to that address, even when it is unhealthy.
// Divide the remaining API budget so a stalled family cannot starve the other.
func serviceHealthAPI(ctx context.Context, client *http.Client, baseURLs []string) (string, []byte, int, error) {
var lastErr, timeoutErr error
for i, baseURL := range baseURLs {
deadline, _ := ctx.Deadline()
candidateCtx, cancel := context.WithTimeout(ctx, time.Until(deadline)/time.Duration(len(baseURLs)-i))
body, status, err := serviceHealthGET(candidateCtx, client, baseURL+"/api/health")
cancel()
if err == nil {
return baseURL, body, status, nil
}
if status != 0 {
// A response whose body fails or exceeds the limit is still an
// observation of this server, not an unavailable address family.
return baseURL, nil, status, err
}
lastErr = err
if serviceHealthErrorCategory(err, "") == telemetry.ServiceHealthFailureTimeout {
timeoutErr = err
}
}
if timeoutErr != nil {
return "", nil, 0, timeoutErr
}
return "", nil, 0, lastErr
}
func serviceHealthErrorCategory(err error, fallback string) string {
var netErr net.Error
if errors.Is(err, context.DeadlineExceeded) || (errors.As(err, &netErr) && netErr.Timeout()) {
return telemetry.ServiceHealthFailureTimeout
}
return fallback
}
func localServiceHealthBaseURLs(listener net.Listener, tlsEnabled bool) []string {
if listener == nil {
return nil
}
tcpAddr, ok := listener.Addr().(*net.TCPAddr)
if !ok || tcpAddr.Port <= 0 || tcpAddr.Port > 65535 {
return nil
}
hosts := []string{tcpAddr.IP.String()}
if tcpAddr.IP == nil || tcpAddr.IP.IsUnspecified() {
hosts = []string{"127.0.0.1"}
if tcpAddr.IP != nil && tcpAddr.IP.To4() == nil {
// Only a proven dual-stack socket owns both loopback families.
// An IPv6-only socket may share its port with an unrelated IPv4
// server. Unknown socket modes stay conservatively IPv6-only.
hosts = []string{"::1"}
if serviceHealthListenerAcceptsIPv4(listener) {
hosts = []string{"127.0.0.1", "::1"}
}
}
} else if tcpAddr.Zone != "" && tcpAddr.IP.To4() == nil {
hosts[0] += "%" + tcpAddr.Zone
}
scheme := "http"
if tlsEnabled {
scheme = "https"
}
baseURLs := make([]string, 0, len(hosts))
for _, host := range hosts {
baseURL := url.URL{Scheme: scheme, Host: net.JoinHostPort(host, fmt.Sprintf("%d", tcpAddr.Port))}
baseURLs = append(baseURLs, baseURL.String())
}
return baseURLs
}
// IPv4-first works when a dual-stack listener serves Pulse over IPv4 but IPv6
// loopback is disabled. Inspect the bound socket rather than inferring its
// mode from the wildcard address or from another server answering on its port.
func serviceHealthListenerAcceptsIPv4(listener net.Listener) bool {
socket, ok := listener.(syscall.Conn)
if !ok {
return false
}
raw, err := socket.SyscallConn()
if err != nil {
return false
}
dualStack := false
if err := raw.Control(func(fd uintptr) {
dualStack = serviceHealthSocketIsDualStack(fd)
}); err != nil {
return false
}
return dualStack
}
func frontendAssetPaths(index []byte) []string {
matches := frontendAssetReferencePattern.FindAllSubmatch(index, serviceHealthAssetLimit)
paths := make([]string, 0, len(matches))
seen := make(map[string]struct{}, len(matches))
for _, match := range matches {
if len(match) != 2 {
continue
}
parsed, err := url.Parse(string(match[1]))
if err != nil || !strings.HasPrefix(parsed.Path, "/assets/") {
continue
}
path := parsed.EscapedPath()
if parsed.RawQuery != "" {
path += "?" + parsed.RawQuery
}
if _, ok := seen[path]; ok {
continue
}
seen[path] = struct{}{}
paths = append(paths, path)
}
return paths
}
func serviceHealthGET(ctx context.Context, client *http.Client, target string) ([]byte, int, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, target, nil)
if err != nil {
return nil, 0, err
}
resp, err := client.Do(req)
if err != nil {
return nil, 0, err
}
defer resp.Body.Close()
body, err := io.ReadAll(io.LimitReader(resp.Body, serviceHealthBodyLimit+1))
if err != nil {
return nil, resp.StatusCode, err
}
if len(body) > serviceHealthBodyLimit {
return nil, resp.StatusCode, fmt.Errorf("service-health response exceeded bounded size")
}
return body, resp.StatusCode, nil
}
func unhealthyServiceObservation(category string) telemetry.ServiceHealthObservation {
return telemetry.ServiceHealthObservation{Observed: true, FailureCategory: category}
}