Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package monitoring
import (
"context"
"encoding/json"
"fmt"
"sort"
"strings"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
monerrors "github.com/rcourtman/pulse-go-rewrite/internal/monitoring/errors"
"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
"github.com/rs/zerolog/log"
)
// isCephStorageType returns true when the provided storage type represents a Ceph backend.
func isCephStorageType(storageType string) bool {
switch strings.ToLower(strings.TrimSpace(storageType)) {
case "rbd", "cephfs", "ceph":
return true
default:
return false
}
}
// pollCephCluster gathers Ceph cluster information when Ceph-backed storage is detected.
func (m *Monitor) pollCephCluster(ctx context.Context, instanceName string, client PVEClientInterface, cephDetected bool) {
if !cephDetected {
// Clear any previously cached Ceph data for this instance.
m.state.UpdateCephClustersForInstance(instanceName, []models.CephCluster{})
return
}
cephCtx := ctx
if deadline, ok := ctx.Deadline(); !ok || time.Until(deadline) > 15*time.Second {
var cancel context.CancelFunc
cephCtx, cancel = context.WithTimeout(ctx, 15*time.Second)
defer cancel()
}
status, err := client.GetCephStatus(cephCtx)
if err != nil {
if monerrors.IsAuthError(err) {
log.Warn().Err(err).Str("instance", instanceName).
Msg("ceph status request denied (401/403) – grant the Pulse token Sys.Audit on path / to enable Ceph monitoring")
} else {
log.Debug().Err(err).Str("instance", instanceName).Msg("ceph status unavailable – preserving previous ceph state")
}
return
}
if status == nil {
log.Debug().Str("instance", instanceName).Msg("ceph status response empty – clearing cached ceph state")
m.state.UpdateCephClustersForInstance(instanceName, []models.CephCluster{})
return
}
df, err := client.GetCephDF(cephCtx)
if err != nil {
log.Debug().Err(err).Str("instance", instanceName).Msg("ceph DF unavailable – continuing with status-only data")
}
cluster := buildCephClusterModel(instanceName, status, df)
if cluster.ID == "" {
// Ensure the cluster has a stable identifier; fall back to instance name.
cluster.ID = instanceName
}
storedClusters := m.state.UpdateCephClustersForInstance(instanceName, []models.CephCluster{cluster})
for _, cluster := range storedClusters {
m.checkCephPoolStorage(cluster)
}
}
func (m *Monitor) checkCephPoolStorage(cluster models.CephCluster) {
storagePools := models.CephPoolStorage(cluster)
if len(storagePools) == 0 {
return
}
timestamp := time.Now()
for _, storage := range storagePools {
if m.metricsHistory != nil && !shouldSkipNativeMockStateMetricWrites() {
m.metricsHistory.AddStorageMetric(storage.ID, "usage", storage.Usage, timestamp)
m.metricsHistory.AddStorageMetric(storage.ID, "used", float64(storage.Used), timestamp)
m.metricsHistory.AddStorageMetric(storage.ID, "total", float64(storage.Total), timestamp)
m.metricsHistory.AddStorageMetric(storage.ID, "avail", float64(storage.Free), timestamp)
if m.metricsStore != nil {
m.metricsStore.Write("storage", storage.ID, "usage", storage.Usage, timestamp)
m.metricsStore.Write("storage", storage.ID, "used", float64(storage.Used), timestamp)
m.metricsStore.Write("storage", storage.ID, "total", float64(storage.Total), timestamp)
m.metricsStore.Write("storage", storage.ID, "avail", float64(storage.Free), timestamp)
}
}
if m.alertManager != nil {
m.alertManager.CheckStorageWithCapacityTrend(storage, m.storageCapacityTrend(storage, timestamp))
}
}
}
// buildCephClusterModel converts the proxmox Ceph responses into the shared model representation.
func buildCephClusterModel(instanceName string, status *proxmox.CephStatus, df *proxmox.CephDF) models.CephCluster {
clusterID := instanceName
if status.FSID != "" {
clusterID = fmt.Sprintf("%s-%s", instanceName, status.FSID)
}
totalBytes := int64(status.PGMap.BytesTotal)
usedBytes := int64(status.PGMap.BytesUsed)
availBytes := int64(status.PGMap.BytesAvail)
if df != nil {
if stats := df.Data.Stats; stats.TotalBytes > 0 {
totalBytes = int64(stats.TotalBytes)
usedBytes = int64(stats.TotalUsedBytes)
availBytes = int64(stats.TotalAvailBytes)
}
}
usagePercent := safePercentage(float64(usedBytes), float64(totalBytes))
pools := make([]models.CephPool, 0)
if df != nil {
for _, pool := range df.Data.Pools {
pools = append(pools, models.CephPool{
ID: pool.ID,
Name: pool.Name,
StoredBytes: int64(pool.Stats.BytesUsed),
AvailableBytes: int64(pool.Stats.MaxAvail),
Objects: int64(pool.Stats.Objects),
PercentUsed: pool.Stats.PercentUsed,
})
}
}
services := make([]models.CephServiceStatus, 0)
if status.ServiceMap.Services != nil {
for serviceType, definition := range status.ServiceMap.Services {
running := 0
total := 0
var offline []string
for daemonName, daemon := range definition.Daemons {
total++
// Standby is a healthy running role for a manager, not an
// offline daemon. Do not extend it to MON/OSD or infer health
// for stopped/unknown daemons from the cluster summary.
if strings.EqualFold(daemon.Status, "running") || strings.EqualFold(daemon.Status, "active") ||
(strings.EqualFold(serviceType, "mgr") && strings.EqualFold(daemon.Status, "standby")) {
running++
continue
}
label := daemonName
if daemon.Host != "" {
label = fmt.Sprintf("%s@%s", daemonName, daemon.Host)
}
offline = append(offline, label)
}
serviceStatus := models.CephServiceStatus{
Type: serviceType,
Running: running,
Total: total,
}
if len(offline) > 0 {
serviceStatus.Message = fmt.Sprintf("Offline: %s", strings.Join(offline, ", "))
}
services = append(services, serviceStatus)
}
}
healthMsg := summarizeCephHealth(status)
healthChecks := cephHealthChecks(status)
numMons := countCephMonitorDaemons(status)
numMgrs := countCephManagerDaemons(status)
cluster := models.CephCluster{
ID: clusterID,
Instance: instanceName,
Source: models.CephClusterSourceProxmoxAPI,
Name: "Ceph",
FSID: status.FSID,
Health: status.Health.Status,
HealthMessage: healthMsg,
HealthChecks: healthChecks,
TotalBytes: totalBytes,
UsedBytes: usedBytes,
AvailableBytes: availBytes,
UsagePercent: usagePercent,
NumMons: numMons,
NumMgrs: numMgrs,
NumOSDs: status.OSDMap.NumOSDs,
NumOSDsUp: status.OSDMap.NumUpOSDs,
NumOSDsIn: status.OSDMap.NumInOSDs,
NumPGs: status.PGMap.NumPGs,
Pools: pools,
Services: services,
LastUpdated: time.Now(),
}
return cluster
}
func cephHealthChecks(status *proxmox.CephStatus) []models.CephHealthCheck {
if status == nil || len(status.Health.Checks) == 0 {
return nil
}
codes := make([]string, 0, len(status.Health.Checks))
for code := range status.Health.Checks {
codes = append(codes, code)
}
sort.Strings(codes)
checks := make([]models.CephHealthCheck, 0, len(codes))
for _, code := range codes {
check := status.Health.Checks[code]
summary := extractCephCheckSummary(check.Summary)
if summary == "" {
for _, detail := range check.Detail {
if strings.TrimSpace(detail.Message) != "" {
summary = strings.TrimSpace(detail.Message)
break
}
}
}
checks = append(checks, models.CephHealthCheck{
Code: strings.TrimSpace(code),
Severity: strings.TrimSpace(check.Severity),
Summary: summary,
})
}
return checks
}
func countCephMonitorDaemons(status *proxmox.CephStatus) int {
if status == nil {
return 0
}
if status.MonMap.NumMons > 0 {
return status.MonMap.NumMons
}
if len(status.MonMap.Mons) > 0 {
return len(status.MonMap.Mons)
}
if len(status.MonMap.Monitors) > 0 {
return len(status.MonMap.Monitors)
}
if len(status.MonMap.QuorumNames) > 0 {
return len(status.MonMap.QuorumNames)
}
if len(status.MonMap.Quorum) > 0 {
return len(status.MonMap.Quorum)
}
// Ceph places quorum membership at the top level of the status payload.
if len(status.QuorumNames) > 0 {
return len(status.QuorumNames)
}
if len(status.Quorum) > 0 {
return len(status.Quorum)
}
return countServiceDaemons(status.ServiceMap.Services, "mon")
}
func countCephManagerDaemons(status *proxmox.CephStatus) int {
if status == nil {
return 0
}
if status.MgrMap.NumMgrs > 0 {
return status.MgrMap.NumMgrs
}
// Quincy+ drops num_mgrs/active_name from mgrmap; derive the count from
// availability plus standbys (num_standbys on modern releases).
active := 0
if status.MgrMap.Available || status.MgrMap.ActiveName != "" {
active = 1
}
standbys := max(status.MgrMap.NumStandbys, len(status.MgrMap.Standbys))
if active+standbys > 0 {
return active + standbys
}
return countServiceDaemons(status.ServiceMap.Services, "mgr")
}
// summarizeCephHealth extracts human-readable messages from the Ceph health payload.
func summarizeCephHealth(status *proxmox.CephStatus) string {
if status == nil {
return ""
}
messages := make([]string, 0)
for _, summary := range status.Health.Summary {
switch {
case summary.Message != "":
messages = append(messages, summary.Message)
case summary.Summary != "":
messages = append(messages, summary.Summary)
}
}
for checkName, check := range status.Health.Checks {
if msg := extractCephCheckSummary(check.Summary); msg != "" {
messages = append(messages, fmt.Sprintf("%s: %s", checkName, msg))
continue
}
for _, detail := range check.Detail {
if detail.Message != "" {
messages = append(messages, fmt.Sprintf("%s: %s", checkName, detail.Message))
break
}
}
}
return strings.Join(messages, "; ")
}
// extractCephCheckSummary attempts to parse the flexible summary field in Ceph health checks into a message string.
func extractCephCheckSummary(raw json.RawMessage) string {
if len(raw) == 0 {
return ""
}
var obj struct {
Message string `json:"message"`
Summary string `json:"summary"`
}
if err := json.Unmarshal(raw, &obj); err == nil {
if obj.Message != "" {
return obj.Message
}
if obj.Summary != "" {
return obj.Summary
}
}
var list []struct {
Message string `json:"message"`
Summary string `json:"summary"`
}
if err := json.Unmarshal(raw, &list); err == nil {
for _, item := range list {
if item.Message != "" {
return item.Message
}
if item.Summary != "" {
return item.Summary
}
}
}
var text string
if err := json.Unmarshal(raw, &text); err == nil {
return text
}
return ""
}
// countServiceDaemons returns the number of daemons defined for a given service type.
func countServiceDaemons(services map[string]proxmox.CephServiceDefinition, serviceType string) int {
if services == nil {
return 0
}
definition, ok := services[serviceType]
if !ok {
return 0
}
return len(definition.Daemons)
}