Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package monitoring
import (
"context"
"fmt"
"strings"
"sync"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/config"
recoverymanager "github.com/rcourtman/pulse-go-rewrite/internal/recovery/manager"
"github.com/rcourtman/pulse-go-rewrite/internal/websocket"
"github.com/rs/zerolog/log"
)
// MultiTenantMonitor manages a dedicated Monitor instance for each organization.
type MultiTenantMonitor struct {
mu sync.RWMutex
monitors map[string]*Monitor
tenantCancel map[string]context.CancelFunc
tenantDone map[string]chan struct{}
tenantDeleting map[string]struct{}
tenantStopping map[string]struct{}
persistence *config.MultiTenantPersistence
baseConfig *config.Config
wsHub *websocket.Hub
recoveryMgr *recoverymanager.Manager
initializer func(*Monitor)
globalCtx context.Context
globalCancel context.CancelFunc
stopped bool // Terminal admission fence, protected by mu.
}
// NewMultiTenantMonitor creates a new multi-tenant monitor manager.
func NewMultiTenantMonitor(baseCfg *config.Config, persistence *config.MultiTenantPersistence, wsHub *websocket.Hub) *MultiTenantMonitor {
ctx, cancel := context.WithCancel(context.Background())
return &MultiTenantMonitor{
monitors: make(map[string]*Monitor),
tenantCancel: make(map[string]context.CancelFunc),
tenantDone: make(map[string]chan struct{}),
tenantDeleting: make(map[string]struct{}),
tenantStopping: make(map[string]struct{}),
persistence: persistence,
baseConfig: baseCfg, // Used as a template or for global settings
wsHub: wsHub,
globalCtx: ctx,
globalCancel: cancel,
}
}
// Both shutdown budgets are variables so tests can shorten them.
var (
// tenantMonitorShutdownTimeout bounds how long process shutdown and
// RemoveTenant wait for a tenant's monitoring loop to exit.
tenantMonitorShutdownTimeout = 2 * time.Second
// tenantDeletionShutdownTimeout bounds how long organization deletion
// waits for the loop. Deletion refuses, and keeps the data, while the loop
// still runs, so this is a correctness gate rather than a best-effort
// wait: a loop that is still starting up or finishing a tick can take
// several seconds to see its cancelled context.
tenantDeletionShutdownTimeout = 30 * time.Second
)
// SetRecoveryManager wires a recovery store manager into all existing and future tenant monitors.
func (mtm *MultiTenantMonitor) SetRecoveryManager(manager *recoverymanager.Manager) {
mtm.mu.Lock()
defer mtm.mu.Unlock()
mtm.recoveryMgr = manager
for _, monitor := range mtm.monitors {
if monitor == nil {
continue
}
monitor.SetRecoveryManager(manager)
}
}
// SetMonitorInitializer configures a callback that is applied to all existing
// and future tenant monitors after creation.
func (mtm *MultiTenantMonitor) SetMonitorInitializer(initializer func(*Monitor)) {
if mtm == nil {
return
}
mtm.mu.Lock()
mtm.initializer = initializer
monitors := make([]*Monitor, 0, len(mtm.monitors))
for _, monitor := range mtm.monitors {
if monitor == nil {
continue
}
monitors = append(monitors, monitor)
}
mtm.mu.Unlock()
if initializer == nil {
return
}
for _, monitor := range monitors {
initializer(monitor)
}
}
// ForEachMonitor runs fn against every live tenant monitor. Use it to
// propagate instance-wide runtime settings (e.g. webhook security allowlist)
// that every org's monitor must observe.
func (mtm *MultiTenantMonitor) ForEachMonitor(fn func(*Monitor)) {
if mtm == nil || fn == nil {
return
}
mtm.mu.RLock()
monitors := make([]*Monitor, 0, len(mtm.monitors))
for _, monitor := range mtm.monitors {
if monitor != nil {
monitors = append(monitors, monitor)
}
}
mtm.mu.RUnlock()
for _, monitor := range monitors {
fn(monitor)
}
}
// ListOrganizationIDs returns the persisted organization IDs known to the
// tenant monitor without requiring every monitor to be initialized first.
func (mtm *MultiTenantMonitor) ListOrganizationIDs() ([]string, error) {
if mtm == nil || mtm.persistence == nil {
return []string{"default"}, nil
}
orgs, err := mtm.persistence.ListOrganizations()
if err != nil {
return nil, err
}
ids := make([]string, 0, len(orgs))
for _, org := range orgs {
if org == nil || strings.TrimSpace(org.ID) == "" {
continue
}
ids = append(ids, strings.TrimSpace(org.ID))
}
if len(ids) == 0 {
ids = append(ids, "default")
}
return ids, nil
}
// GetMonitor returns the monitor instance for a specific organization.
// It lazily initializes the monitor if it doesn't exist.
func (mtm *MultiTenantMonitor) GetMonitor(orgID string) (*Monitor, error) {
orgID = strings.TrimSpace(orgID)
if orgID == "" {
return nil, fmt.Errorf("organization ID is required")
}
mtm.mu.RLock()
if mtm.stopped {
mtm.mu.RUnlock()
return nil, fmt.Errorf("tenant monitor manager is stopped")
}
monitor, exists := mtm.monitors[orgID]
_, deleting := mtm.tenantDeleting[orgID]
mtm.mu.RUnlock()
if deleting {
return nil, fmt.Errorf("organization %q is being deleted", orgID)
}
if exists {
return monitor, nil
}
mtm.mu.Lock()
defer mtm.mu.Unlock()
// Stop can win between the cache lookup and this initialization lock.
// A retired manager must never reopen stores or mutate the shared config.
if mtm.stopped {
return nil, fmt.Errorf("tenant monitor manager is stopped")
}
if mtm.monitors == nil {
mtm.monitors = make(map[string]*Monitor)
}
if mtm.tenantCancel == nil {
mtm.tenantCancel = make(map[string]context.CancelFunc)
}
if mtm.tenantDone == nil {
mtm.tenantDone = make(map[string]chan struct{})
}
if mtm.tenantDeleting == nil {
mtm.tenantDeleting = make(map[string]struct{})
}
// Double-check locking pattern
if _, deleting = mtm.tenantDeleting[orgID]; deleting {
return nil, fmt.Errorf("organization %q is being deleted", orgID)
}
if monitor, exists = mtm.monitors[orgID]; exists {
return monitor, nil
}
if mtm.persistence == nil {
return nil, fmt.Errorf("tenant persistence is not configured")
}
if orgID != "default" && !mtm.persistence.OrgExists(orgID) {
return nil, fmt.Errorf("organization %q is not provisioned", orgID)
}
// Initialize new monitor for this tenant
log.Info().Str("org_id", orgID).Msg("initializing tenant monitor")
// 1. Load Tenant Config
// The default org is the primary runtime and must retain the canonical
// configuration pointer owned by the server. Runtime auth mutations such as
// freshly minted agent tokens need to become visible to monitoring-backed
// diagnostics immediately. Non-default tenants remain isolated deep copies
// so their mutable state and credentials cannot alias the primary runtime.
tenantConfig := mtm.baseConfig
// Resolve scoped persistence before loading any resource or credential.
tenantPersistence, err := mtm.persistence.GetPersistence(orgID)
if err != nil {
return nil, fmt.Errorf("failed to get persistence for org %s: %w", orgID, err)
}
var instancePersistence *config.ConfigPersistence
if orgID != "default" {
tenantConfig, err = loadTenantRuntimeConfig(mtm.baseConfig, tenantPersistence)
if err != nil {
return nil, fmt.Errorf("failed to load configuration for org %s: %w", orgID, err)
}
// The org's own config copy holds none of the instance tokens bound
// to it, which is where its agent install tokens live.
instancePersistence, err = mtm.persistence.GetPersistence("default")
if err != nil {
return nil, fmt.Errorf("failed to get instance persistence for org %s: %w", orgID, err)
}
} else {
tenantConfig.PVEInstances = nil
tenantConfig.PBSInstances = nil
tenantConfig.PMGInstances = nil
tenantConfig.DataPath = tenantPersistence.GetConfigDir()
if nodes, loadErr := tenantPersistence.LoadNodesConfig(); loadErr != nil {
log.Warn().Err(loadErr).Str("org_id", orgID).Msg("failed to load tenant nodes config, starting with empty config")
} else if nodes != nil {
tenantConfig.PVEInstances = nodes.PVEInstances
tenantConfig.PBSInstances = nodes.PBSInstances
tenantConfig.PMGInstances = nodes.PMGInstances
}
}
// 2. Create Monitor
// Usage of internal New constructor
monitor, err = New(tenantConfig)
if err != nil {
return nil, fmt.Errorf("failed to create monitor for org %s: %w", orgID, err)
}
tenantPersistence.SetMetadataStores(
monitor.GuestMetadataStore(),
monitor.DockerMetadataStore(),
monitor.HostMetadataStore(),
)
// Set org ID for tenant isolation
// This enables tenant-scoped WebSocket broadcasts
monitor.SetOrgID(orgID)
if instancePersistence != nil {
monitor.SetInstanceAPITokenInventory(mtm.baseConfig, instancePersistence)
}
// Stamp this org's identity into webhook payloads so external receivers
// (PSA/ticketing bridges) can route by tenant. Resolved lazily so a
// display-name rename is picked up without restarting the monitor. The
// default org keeps the PULSE_TENANT_ID/PULSE_TENANT_NAME env defaults.
if nm := monitor.GetNotificationManager(); nm != nil && orgID != "default" {
persistence := mtm.persistence
nm.SetTenantIdentityResolver(func() (string, string) {
name := ""
if org, err := persistence.LoadOrganization(orgID); err == nil && org != nil {
name = org.DisplayName
}
return orgID, name
})
}
if mtm.recoveryMgr != nil {
monitor.SetRecoveryManager(mtm.recoveryMgr)
}
if mtm.initializer != nil {
mtm.initializer(monitor)
}
// 3. Start Monitor with a tenant-scoped runtime so RemoveTenant can stop it cleanly.
tenantCtx, tenantCancel := context.WithCancel(mtm.globalCtx)
tenantDone := make(chan struct{})
go func() {
defer close(tenantDone)
monitor.Start(tenantCtx, mtm.wsHub)
}()
mtm.monitors[orgID] = monitor
mtm.tenantCancel[orgID] = tenantCancel
mtm.tenantDone[orgID] = tenantDone
return monitor, nil
}
// PeekMonitor returns the tenant monitor instance if it is already initialized.
// It does not create a new monitor.
func (mtm *MultiTenantMonitor) PeekMonitor(orgID string) (*Monitor, bool) {
orgID = strings.TrimSpace(orgID)
if orgID == "" {
return nil, false
}
mtm.mu.RLock()
defer mtm.mu.RUnlock()
if mtm.stopped {
return nil, false
}
monitor, exists := mtm.monitors[orgID]
return monitor, exists
}
// Stop permanently closes admission and stops all tenant monitors. A new
// manager, rather than a lookup on this retired one, owns replacement runtimes.
func (mtm *MultiTenantMonitor) Stop() {
log.Info().Msg("stopping MultiTenantMonitor and all tenant instances")
mtm.mu.Lock()
mtm.stopped = true
if mtm.globalCancel != nil {
mtm.globalCancel()
}
orgIDs := make([]string, 0, len(mtm.monitors))
for orgID := range mtm.monitors {
orgIDs = append(orgIDs, orgID)
}
mtm.mu.Unlock()
for _, orgID := range orgIDs {
if err := mtm.removeTenant(orgID, false); err != nil {
log.Error().Err(err).Str("org_id", orgID).Msg("tenant shutdown incomplete; retaining runtime ownership and data")
}
}
}
// RemoveTenant stops and removes a specific tenant's monitor.
// Useful for offboarding or manual reloading.
func (mtm *MultiTenantMonitor) RemoveTenant(orgID string) {
if err := mtm.removeTenant(orgID, false); err != nil {
log.Error().Err(err).Str("org_id", orgID).Msg("tenant removal incomplete; retaining runtime ownership and data")
}
}
// BeginTenantDeletion prevents lazy monitor initialization while a tenant's
// persistence is being removed. Call FinishTenantDeletion after the
// persistence operation completes. An incomplete shutdown retains its guard.
func (mtm *MultiTenantMonitor) BeginTenantDeletion(orgID string) error {
return mtm.removeTenant(orgID, true)
}
// FinishTenantDeletion releases the temporary lifecycle guard installed by
// BeginTenantDeletion. Once persistence has been removed, GetMonitor remains
// unable to recreate the tenant because the organization no longer exists.
func (mtm *MultiTenantMonitor) FinishTenantDeletion(orgID string) {
orgID = strings.TrimSpace(orgID)
if orgID == "" {
return
}
mtm.mu.Lock()
// An incomplete removal still owns a runtime or writer. Do not release
// its guard merely because an offboarding caller ran deferred cleanup.
if _, retained := mtm.monitors[orgID]; !retained {
delete(mtm.tenantDeleting, orgID)
}
mtm.mu.Unlock()
}
func (mtm *MultiTenantMonitor) removeTenant(orgID string, deleting bool) error {
orgID = strings.TrimSpace(orgID)
if orgID == "" {
return nil
}
mtm.mu.Lock()
if mtm.tenantDeleting == nil {
mtm.tenantDeleting = make(map[string]struct{})
}
if mtm.tenantStopping == nil {
mtm.tenantStopping = make(map[string]struct{})
}
if _, stopping := mtm.tenantStopping[orgID]; stopping {
mtm.mu.Unlock()
return fmt.Errorf("organization %q shutdown is already in progress", orgID)
}
mtm.tenantDeleting[orgID] = struct{}{}
mtm.tenantStopping[orgID] = struct{}{}
monitor := mtm.monitors[orgID]
cancel := mtm.tenantCancel[orgID]
done := mtm.tenantDone[orgID]
mtm.mu.Unlock()
defer func() {
mtm.mu.Lock()
delete(mtm.tenantStopping, orgID)
mtm.mu.Unlock()
}()
log.Info().Str("org_id", orgID).Msg("stopping and removing tenant monitor")
if cancel != nil {
cancel()
}
shutdownTimeout := tenantMonitorShutdownTimeout
if deleting {
shutdownTimeout = tenantDeletionShutdownTimeout
}
if !waitForTenantMonitorShutdown(orgID, done, shutdownTimeout) {
return fmt.Errorf("organization %q monitoring loop has not stopped", orgID)
}
if monitor != nil {
if err := monitor.stop(); err != nil {
return fmt.Errorf("organization %q guest metadata shutdown incomplete: %w", orgID, err)
}
}
mtm.mu.Lock()
delete(mtm.monitors, orgID)
delete(mtm.tenantCancel, orgID)
delete(mtm.tenantDone, orgID)
if !deleting {
delete(mtm.tenantDeleting, orgID)
}
mtm.mu.Unlock()
return nil
}
func waitForTenantMonitorShutdown(orgID string, done <-chan struct{}, timeout time.Duration) bool {
if done == nil {
return true
}
timer := time.NewTimer(timeout)
defer timer.Stop()
select {
case <-done:
return true
case <-timer.C:
logger := log.Warn().Dur("timeout", timeout)
if orgID != "" {
logger = logger.Str("org_id", orgID)
}
logger.Msg("timed out waiting for tenant monitor loop to exit")
return false
}
}
// OrgExists checks if an organization exists (directory exists) without creating it.
func (mtm *MultiTenantMonitor) OrgExists(orgID string) bool {
orgID = strings.TrimSpace(orgID)
if orgID == "" {
return false
}
if mtm.persistence == nil {
return false
}
return mtm.persistence.OrgExists(orgID)
}