Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

14.2 KB
Raw Blame
package monitoring

import (
	"context"
	"fmt"
	"strings"
	"sync"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/config"
	recoverymanager "github.com/rcourtman/pulse-go-rewrite/internal/recovery/manager"
	"github.com/rcourtman/pulse-go-rewrite/internal/websocket"
	"github.com/rs/zerolog/log"
)

// MultiTenantMonitor manages a dedicated Monitor instance for each organization.
type MultiTenantMonitor struct {
	mu             sync.RWMutex
	monitors       map[string]*Monitor
	tenantCancel   map[string]context.CancelFunc
	tenantDone     map[string]chan struct{}
	tenantDeleting map[string]struct{}
	tenantStopping map[string]struct{}
	persistence    *config.MultiTenantPersistence
	baseConfig     *config.Config
	wsHub          *websocket.Hub
	recoveryMgr    *recoverymanager.Manager
	initializer    func(*Monitor)
	globalCtx      context.Context
	globalCancel   context.CancelFunc
	stopped        bool // Terminal admission fence, protected by mu.
}

// NewMultiTenantMonitor creates a new multi-tenant monitor manager.
func NewMultiTenantMonitor(baseCfg *config.Config, persistence *config.MultiTenantPersistence, wsHub *websocket.Hub) *MultiTenantMonitor {
	ctx, cancel := context.WithCancel(context.Background())
	return &MultiTenantMonitor{
		monitors:       make(map[string]*Monitor),
		tenantCancel:   make(map[string]context.CancelFunc),
		tenantDone:     make(map[string]chan struct{}),
		tenantDeleting: make(map[string]struct{}),
		tenantStopping: make(map[string]struct{}),
		persistence:    persistence,
		baseConfig:     baseCfg, // Used as a template or for global settings
		wsHub:          wsHub,
		globalCtx:      ctx,
		globalCancel:   cancel,
	}
}

// Both shutdown budgets are variables so tests can shorten them.
var (
	// tenantMonitorShutdownTimeout bounds how long process shutdown and
	// RemoveTenant wait for a tenant's monitoring loop to exit.
	tenantMonitorShutdownTimeout = 2 * time.Second

	// tenantDeletionShutdownTimeout bounds how long organization deletion
	// waits for the loop. Deletion refuses, and keeps the data, while the loop
	// still runs, so this is a correctness gate rather than a best-effort
	// wait: a loop that is still starting up or finishing a tick can take
	// several seconds to see its cancelled context.
	tenantDeletionShutdownTimeout = 30 * time.Second
)

// SetRecoveryManager wires a recovery store manager into all existing and future tenant monitors.
func (mtm *MultiTenantMonitor) SetRecoveryManager(manager *recoverymanager.Manager) {
	mtm.mu.Lock()
	defer mtm.mu.Unlock()
	mtm.recoveryMgr = manager
	for _, monitor := range mtm.monitors {
		if monitor == nil {
			continue
		}
		monitor.SetRecoveryManager(manager)
	}
}

// SetMonitorInitializer configures a callback that is applied to all existing
// and future tenant monitors after creation.
func (mtm *MultiTenantMonitor) SetMonitorInitializer(initializer func(*Monitor)) {
	if mtm == nil {
		return
	}

	mtm.mu.Lock()
	mtm.initializer = initializer
	monitors := make([]*Monitor, 0, len(mtm.monitors))
	for _, monitor := range mtm.monitors {
		if monitor == nil {
			continue
		}
		monitors = append(monitors, monitor)
	}
	mtm.mu.Unlock()

	if initializer == nil {
		return
	}
	for _, monitor := range monitors {
		initializer(monitor)
	}
}

// ForEachMonitor runs fn against every live tenant monitor. Use it to
// propagate instance-wide runtime settings (e.g. webhook security allowlist)
// that every org's monitor must observe.
func (mtm *MultiTenantMonitor) ForEachMonitor(fn func(*Monitor)) {
	if mtm == nil || fn == nil {
		return
	}
	mtm.mu.RLock()
	monitors := make([]*Monitor, 0, len(mtm.monitors))
	for _, monitor := range mtm.monitors {
		if monitor != nil {
			monitors = append(monitors, monitor)
		}
	}
	mtm.mu.RUnlock()
	for _, monitor := range monitors {
		fn(monitor)
	}
}

// ListOrganizationIDs returns the persisted organization IDs known to the
// tenant monitor without requiring every monitor to be initialized first.
func (mtm *MultiTenantMonitor) ListOrganizationIDs() ([]string, error) {
	if mtm == nil || mtm.persistence == nil {
		return []string{"default"}, nil
	}
	orgs, err := mtm.persistence.ListOrganizations()
	if err != nil {
		return nil, err
	}
	ids := make([]string, 0, len(orgs))
	for _, org := range orgs {
		if org == nil || strings.TrimSpace(org.ID) == "" {
			continue
		}
		ids = append(ids, strings.TrimSpace(org.ID))
	}
	if len(ids) == 0 {
		ids = append(ids, "default")
	}
	return ids, nil
}

// GetMonitor returns the monitor instance for a specific organization.
// It lazily initializes the monitor if it doesn't exist.
func (mtm *MultiTenantMonitor) GetMonitor(orgID string) (*Monitor, error) {
	orgID = strings.TrimSpace(orgID)
	if orgID == "" {
		return nil, fmt.Errorf("organization ID is required")
	}

	mtm.mu.RLock()
	if mtm.stopped {
		mtm.mu.RUnlock()
		return nil, fmt.Errorf("tenant monitor manager is stopped")
	}
	monitor, exists := mtm.monitors[orgID]
	_, deleting := mtm.tenantDeleting[orgID]
	mtm.mu.RUnlock()

	if deleting {
		return nil, fmt.Errorf("organization %q is being deleted", orgID)
	}
	if exists {
		return monitor, nil
	}

	mtm.mu.Lock()
	defer mtm.mu.Unlock()

	// Stop can win between the cache lookup and this initialization lock.
	// A retired manager must never reopen stores or mutate the shared config.
	if mtm.stopped {
		return nil, fmt.Errorf("tenant monitor manager is stopped")
	}

	if mtm.monitors == nil {
		mtm.monitors = make(map[string]*Monitor)
	}
	if mtm.tenantCancel == nil {
		mtm.tenantCancel = make(map[string]context.CancelFunc)
	}
	if mtm.tenantDone == nil {
		mtm.tenantDone = make(map[string]chan struct{})
	}
	if mtm.tenantDeleting == nil {
		mtm.tenantDeleting = make(map[string]struct{})
	}

	// Double-check locking pattern
	if _, deleting = mtm.tenantDeleting[orgID]; deleting {
		return nil, fmt.Errorf("organization %q is being deleted", orgID)
	}
	if monitor, exists = mtm.monitors[orgID]; exists {
		return monitor, nil
	}

	if mtm.persistence == nil {
		return nil, fmt.Errorf("tenant persistence is not configured")
	}
	if orgID != "default" && !mtm.persistence.OrgExists(orgID) {
		return nil, fmt.Errorf("organization %q is not provisioned", orgID)
	}

	// Initialize new monitor for this tenant
	log.Info().Str("org_id", orgID).Msg("initializing tenant monitor")

	// 1. Load Tenant Config
	// The default org is the primary runtime and must retain the canonical
	// configuration pointer owned by the server. Runtime auth mutations such as
	// freshly minted agent tokens need to become visible to monitoring-backed
	// diagnostics immediately. Non-default tenants remain isolated deep copies
	// so their mutable state and credentials cannot alias the primary runtime.
	tenantConfig := mtm.baseConfig

	// Resolve scoped persistence before loading any resource or credential.
	tenantPersistence, err := mtm.persistence.GetPersistence(orgID)
	if err != nil {
		return nil, fmt.Errorf("failed to get persistence for org %s: %w", orgID, err)
	}
	var instancePersistence *config.ConfigPersistence
	if orgID != "default" {
		tenantConfig, err = loadTenantRuntimeConfig(mtm.baseConfig, tenantPersistence)
		if err != nil {
			return nil, fmt.Errorf("failed to load configuration for org %s: %w", orgID, err)
		}
		// The org's own config copy holds none of the instance tokens bound
		// to it, which is where its agent install tokens live.
		instancePersistence, err = mtm.persistence.GetPersistence("default")
		if err != nil {
			return nil, fmt.Errorf("failed to get instance persistence for org %s: %w", orgID, err)
		}
	} else {
		tenantConfig.PVEInstances = nil
		tenantConfig.PBSInstances = nil
		tenantConfig.PMGInstances = nil
		tenantConfig.DataPath = tenantPersistence.GetConfigDir()
		if nodes, loadErr := tenantPersistence.LoadNodesConfig(); loadErr != nil {
			log.Warn().Err(loadErr).Str("org_id", orgID).Msg("failed to load tenant nodes config, starting with empty config")
		} else if nodes != nil {
			tenantConfig.PVEInstances = nodes.PVEInstances
			tenantConfig.PBSInstances = nodes.PBSInstances
			tenantConfig.PMGInstances = nodes.PMGInstances
		}
	}

	// 2. Create Monitor
	// Usage of internal New constructor
	monitor, err = New(tenantConfig)
	if err != nil {
		return nil, fmt.Errorf("failed to create monitor for org %s: %w", orgID, err)
	}
	tenantPersistence.SetMetadataStores(
		monitor.GuestMetadataStore(),
		monitor.DockerMetadataStore(),
		monitor.HostMetadataStore(),
	)

	// Set org ID for tenant isolation
	// This enables tenant-scoped WebSocket broadcasts
	monitor.SetOrgID(orgID)
	if instancePersistence != nil {
		monitor.SetInstanceAPITokenInventory(mtm.baseConfig, instancePersistence)
	}

	// Stamp this org's identity into webhook payloads so external receivers
	// (PSA/ticketing bridges) can route by tenant. Resolved lazily so a
	// display-name rename is picked up without restarting the monitor. The
	// default org keeps the PULSE_TENANT_ID/PULSE_TENANT_NAME env defaults.
	if nm := monitor.GetNotificationManager(); nm != nil && orgID != "default" {
		persistence := mtm.persistence
		nm.SetTenantIdentityResolver(func() (string, string) {
			name := ""
			if org, err := persistence.LoadOrganization(orgID); err == nil && org != nil {
				name = org.DisplayName
			}
			return orgID, name
		})
	}
	if mtm.recoveryMgr != nil {
		monitor.SetRecoveryManager(mtm.recoveryMgr)
	}
	if mtm.initializer != nil {
		mtm.initializer(monitor)
	}

	// 3. Start Monitor with a tenant-scoped runtime so RemoveTenant can stop it cleanly.
	tenantCtx, tenantCancel := context.WithCancel(mtm.globalCtx)
	tenantDone := make(chan struct{})
	go func() {
		defer close(tenantDone)
		monitor.Start(tenantCtx, mtm.wsHub)
	}()

	mtm.monitors[orgID] = monitor
	mtm.tenantCancel[orgID] = tenantCancel
	mtm.tenantDone[orgID] = tenantDone
	return monitor, nil
}

// PeekMonitor returns the tenant monitor instance if it is already initialized.
// It does not create a new monitor.
func (mtm *MultiTenantMonitor) PeekMonitor(orgID string) (*Monitor, bool) {
	orgID = strings.TrimSpace(orgID)
	if orgID == "" {
		return nil, false
	}

	mtm.mu.RLock()
	defer mtm.mu.RUnlock()
	if mtm.stopped {
		return nil, false
	}
	monitor, exists := mtm.monitors[orgID]
	return monitor, exists
}

// Stop permanently closes admission and stops all tenant monitors. A new
// manager, rather than a lookup on this retired one, owns replacement runtimes.
func (mtm *MultiTenantMonitor) Stop() {
	log.Info().Msg("stopping MultiTenantMonitor and all tenant instances")

	mtm.mu.Lock()
	mtm.stopped = true
	if mtm.globalCancel != nil {
		mtm.globalCancel()
	}
	orgIDs := make([]string, 0, len(mtm.monitors))
	for orgID := range mtm.monitors {
		orgIDs = append(orgIDs, orgID)
	}
	mtm.mu.Unlock()
	for _, orgID := range orgIDs {
		if err := mtm.removeTenant(orgID, false); err != nil {
			log.Error().Err(err).Str("org_id", orgID).Msg("tenant shutdown incomplete; retaining runtime ownership and data")
		}
	}
}

// RemoveTenant stops and removes a specific tenant's monitor.
// Useful for offboarding or manual reloading.
func (mtm *MultiTenantMonitor) RemoveTenant(orgID string) {
	if err := mtm.removeTenant(orgID, false); err != nil {
		log.Error().Err(err).Str("org_id", orgID).Msg("tenant removal incomplete; retaining runtime ownership and data")
	}
}

// BeginTenantDeletion prevents lazy monitor initialization while a tenant's
// persistence is being removed. Call FinishTenantDeletion after the
// persistence operation completes. An incomplete shutdown retains its guard.
func (mtm *MultiTenantMonitor) BeginTenantDeletion(orgID string) error {
	return mtm.removeTenant(orgID, true)
}

// FinishTenantDeletion releases the temporary lifecycle guard installed by
// BeginTenantDeletion. Once persistence has been removed, GetMonitor remains
// unable to recreate the tenant because the organization no longer exists.
func (mtm *MultiTenantMonitor) FinishTenantDeletion(orgID string) {
	orgID = strings.TrimSpace(orgID)
	if orgID == "" {
		return
	}

	mtm.mu.Lock()
	// An incomplete removal still owns a runtime or writer. Do not release
	// its guard merely because an offboarding caller ran deferred cleanup.
	if _, retained := mtm.monitors[orgID]; !retained {
		delete(mtm.tenantDeleting, orgID)
	}
	mtm.mu.Unlock()
}

func (mtm *MultiTenantMonitor) removeTenant(orgID string, deleting bool) error {
	orgID = strings.TrimSpace(orgID)
	if orgID == "" {
		return nil
	}

	mtm.mu.Lock()
	if mtm.tenantDeleting == nil {
		mtm.tenantDeleting = make(map[string]struct{})
	}
	if mtm.tenantStopping == nil {
		mtm.tenantStopping = make(map[string]struct{})
	}
	if _, stopping := mtm.tenantStopping[orgID]; stopping {
		mtm.mu.Unlock()
		return fmt.Errorf("organization %q shutdown is already in progress", orgID)
	}
	mtm.tenantDeleting[orgID] = struct{}{}
	mtm.tenantStopping[orgID] = struct{}{}
	monitor := mtm.monitors[orgID]
	cancel := mtm.tenantCancel[orgID]
	done := mtm.tenantDone[orgID]
	mtm.mu.Unlock()
	defer func() {
		mtm.mu.Lock()
		delete(mtm.tenantStopping, orgID)
		mtm.mu.Unlock()
	}()

	log.Info().Str("org_id", orgID).Msg("stopping and removing tenant monitor")
	if cancel != nil {
		cancel()
	}
	shutdownTimeout := tenantMonitorShutdownTimeout
	if deleting {
		shutdownTimeout = tenantDeletionShutdownTimeout
	}
	if !waitForTenantMonitorShutdown(orgID, done, shutdownTimeout) {
		return fmt.Errorf("organization %q monitoring loop has not stopped", orgID)
	}
	if monitor != nil {
		if err := monitor.stop(); err != nil {
			return fmt.Errorf("organization %q guest metadata shutdown incomplete: %w", orgID, err)
		}
	}

	mtm.mu.Lock()
	delete(mtm.monitors, orgID)
	delete(mtm.tenantCancel, orgID)
	delete(mtm.tenantDone, orgID)
	if !deleting {
		delete(mtm.tenantDeleting, orgID)
	}
	mtm.mu.Unlock()
	return nil
}

func waitForTenantMonitorShutdown(orgID string, done <-chan struct{}, timeout time.Duration) bool {
	if done == nil {
		return true
	}

	timer := time.NewTimer(timeout)
	defer timer.Stop()

	select {
	case <-done:
		return true
	case <-timer.C:
		logger := log.Warn().Dur("timeout", timeout)
		if orgID != "" {
			logger = logger.Str("org_id", orgID)
		}
		logger.Msg("timed out waiting for tenant monitor loop to exit")
		return false
	}
}

// OrgExists checks if an organization exists (directory exists) without creating it.
func (mtm *MultiTenantMonitor) OrgExists(orgID string) bool {
	orgID = strings.TrimSpace(orgID)
	if orgID == "" {
		return false
	}

	if mtm.persistence == nil {
		return false
	}
	return mtm.persistence.OrgExists(orgID)
}