Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package monitoring
import (
"strings"
"sync"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
"github.com/rcourtman/pulse-go-rewrite/internal/mock"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
"github.com/rs/zerolog/log"
)
const backupIntentEvidenceMaxAge = 5 * time.Minute
type resourceOperatorIntentReader interface {
GetResourceOperatorState(canonicalID string) (unifiedresources.ResourceOperatorState, bool, error)
}
type resourceIntentIdentityReader interface {
ResolveCanonicalResourceID(ref string) (string, bool)
}
type resourceIntentAncestorReader interface {
ResolveCanonicalResourceAncestors(ref string) []string
}
func (m *Monitor) installOperatorIntentResolver(store ResourceStoreInterface) {
if m == nil || m.alertManager == nil {
return
}
identity := m.newOperatorIntentIdentity(store)
m.operatorIntentIdentity.Store(identity)
identity.refresh()
identityStore := func() any {
if identity == nil {
return store
}
return identity.current()
}
identityReader, hasIdentityReader := store.(resourceIntentIdentityReader)
if hasIdentityReader && identityReader != nil {
m.alertManager.SetResourceIntentIdentityResolver(func(ref string) (string, bool) {
return identityStore().(resourceIntentIdentityReader).ResolveCanonicalResourceID(ref)
})
} else {
m.alertManager.SetResourceIntentIdentityResolver(nil)
}
reader, ok := store.(resourceOperatorIntentReader)
if !ok || reader == nil {
m.alertManager.SetOperatorIntentContextResolver(nil)
return
}
_, hasAncestorReader := store.(resourceIntentAncestorReader)
m.alertManager.SetOperatorIntentContextResolver(func(resourceID string, now time.Time) (alerts.OperatorIntentContext, bool) {
identity := identityStore()
if hasIdentityReader {
if canonicalID, found := identity.(resourceIntentIdentityReader).ResolveCanonicalResourceID(resourceID); found {
resourceID = canonicalID
}
}
state, found, err := reader.GetResourceOperatorState(resourceID)
if err != nil {
log.Warn().Err(err).Str("resourceID", resourceID).Msg("Failed to read operator state for alert intent")
return alerts.OperatorIntentContext{}, false
}
context := alerts.OperatorIntentContext{}
if found {
context = alerts.OperatorIntentContext{
IntentionallyOffline: state.IntentionallyOffline,
MonitoringMode: string(state.MonitoringMode),
LifecycleState: string(state.LifecycleState),
}
}
var own *unifiedresources.ResourceOperatorState
if found {
own = &state
}
var ancestorIDs []string
if hasAncestorReader {
ancestorIDs = identity.(resourceIntentAncestorReader).ResolveCanonicalResourceAncestors(resourceID)
}
coverage, covered := unifiedresources.ActiveMaintenanceCoverageAt(now, resourceID, own, ancestorIDs, func(ancestorID string) (unifiedresources.ResourceOperatorState, bool) {
ancestorState, ancestorFound, ancestorErr := reader.GetResourceOperatorState(ancestorID)
if ancestorErr != nil {
log.Warn().Err(ancestorErr).Str("resourceID", resourceID).Str("ancestorID", ancestorID).Msg("Failed to read inherited operator maintenance state")
return unifiedresources.ResourceOperatorState{}, false
}
return ancestorState, ancestorFound
})
if covered {
startAt, endAt := coverage.Occurrence.StartAt, coverage.Occurrence.EndAt
context.MaintenanceStartAt = &startAt
context.MaintenanceEndAt = &endAt
context.MaintenanceReason = coverage.Reason
context.MaintenanceSourceID = coverage.SourceID
context.MaintenanceInherited = coverage.Inherited
context.MaintenanceScope = string(coverage.Scope)
}
return context, found || covered
})
// Alerts restore before the resource store is attached during startup.
// Reconcile that restored set as soon as persisted policy is available.
if m.alertManager.ReconcileOperatorIntentState() > 0 && m.state != nil {
m.SyncAlertState()
}
}
// operatorIntentIdentity is the store alert intent resolves resource
// references and ancestors through: the published registry with saved hosts
// that have not reported since a restart overlaid, the read state Patrol
// resolves through and the resources API seeds from. A saved host exists only
// there, and an operator link holds it under the link's primary, so the
// published registry alone left the saved host's alerts reading their literal
// reference and missed intent set on that host or on the primary.
//
// The alert manager resolves under its own lock, once per metric check, so
// the overlay is kept for the published generation it was built from.
// Publication builds it before evaluating alerts (refresh, outside that
// lock); a lookup that races ahead of publication builds it itself. A
// saved-host change between generations shows on the next one. Mock mode
// carries no saved hosts: the published registry answers and any real-mode
// overlay is dropped.
type operatorIntentIdentity struct {
monitor *Monitor
store ResourceStoreInterface
published unifiedresources.ReadState
generations interface {
Generation() unifiedresources.RegistryGeneration
}
mu sync.Mutex
built bool
builtFor unifiedresources.RegistryGeneration
readState any
}
// newOperatorIntentIdentity returns nil for a store without generations or
// the reference and ancestor readers, which alert intent then uses as is.
func (m *Monitor) newOperatorIntentIdentity(store ResourceStoreInterface) *operatorIntentIdentity {
published, isReadState := store.(unifiedresources.ReadState)
generations, versioned := store.(interface {
Generation() unifiedresources.RegistryGeneration
})
_, resolves := store.(resourceIntentIdentityReader)
_, walksAncestors := store.(resourceIntentAncestorReader)
if !isReadState || !versioned || !resolves || !walksAncestors {
return nil
}
return &operatorIntentIdentity{monitor: m, store: store, published: published, generations: generations}
}
// current returns the read state for the current published generation.
func (i *operatorIntentIdentity) current() any {
if mock.IsMockEnabled() {
i.mu.Lock()
i.built, i.readState = false, nil
i.mu.Unlock()
return i.store
}
generation := i.generations.Generation()
i.mu.Lock()
if i.built && i.builtFor == generation {
readState := i.readState
i.mu.Unlock()
return readState
}
i.mu.Unlock()
return i.build(generation)
}
// refresh builds the read state for the current published generation unless
// it is already built.
func (i *operatorIntentIdentity) refresh() {
if i == nil || mock.IsMockEnabled() {
return
}
generation := i.generations.Generation()
i.mu.Lock()
fresh := i.built && i.builtFor == generation
i.mu.Unlock()
if !fresh {
i.build(generation)
}
}
// build overlays saved hosts on the published registry outside i.mu, so a
// concurrent lookup waits on neither the overlay nor the store reads it makes.
// It keeps the result only while the mock-mode epoch it started in and the
// generation it read are still current. The check and the store run inside
// the fence, so a mode switch, which advances the fence after flipping the
// mode, either waits for them or makes them refuse: a build that read no
// saved hosts because mock mode was on never stands in for real mode.
func (i *operatorIntentIdentity) build(generation unifiedresources.RegistryGeneration) any {
scope := i.monitor.mockModeFence.begin()
if mock.IsMockEnabled() {
return i.store
}
readState := any(i.store)
overlay := i.monitor.readStateWithStandaloneHostContinuity(i.published)
if _, ok := overlay.(resourceIntentIdentityReader); ok {
if _, ok := overlay.(resourceIntentAncestorReader); ok {
readState = overlay
}
}
scope.run(func() {
if mock.IsMockEnabled() || i.generations.Generation() != generation {
return
}
i.mu.Lock()
i.built, i.builtFor, i.readState = true, generation, readState
i.mu.Unlock()
})
return readState
}
func (m *Monitor) resolveBackupIntentContext(_ string, instance, node string, vmid int, now time.Time) (alerts.BackupIntentContext, bool) {
if m == nil || vmid <= 0 {
return alerts.BackupIntentContext{}, false
}
m.mu.RLock()
state := m.state
m.mu.RUnlock()
if state == nil {
return alerts.BackupIntentContext{}, false
}
instance = strings.TrimSpace(instance)
node = strings.TrimSpace(node)
if now.IsZero() {
now = time.Now().UTC()
}
// Policy needs only these primary task records, not every guest, sensor,
// backup artifact and Kubernetes object in the monitored estate.
for _, task := range state.GetBackupTasks() {
if task.VMID != vmid || (instance != "" && task.Instance != instance) || (node != "" && task.Node != "" && task.Node != node) {
continue
}
if task.ObservedAt.IsZero() || now.Sub(task.ObservedAt) > backupIntentEvidenceMaxAge || task.ObservedAt.After(now.Add(time.Minute)) {
continue
}
status := strings.ToLower(strings.TrimSpace(task.Status))
if !task.EndTime.IsZero() || status == "ok" || status == "stopped" || status == "error" || status == "warning" {
continue
}
return alerts.BackupIntentContext{
Active: true,
ObservedAt: task.ObservedAt,
Evidence: "pve_vzdump_task:" + task.ID,
}, true
}
return alerts.BackupIntentContext{}, false
}