Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

6.3 KB
Raw Blame
package monitoring

import (
	"strings"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
)

// MonitoredSystemUsageSnapshot describes whether the monitor can currently
// supply a canonical monitored-system count that is safe for settings and
// support context to consume.
type MonitoredSystemUsageSnapshot struct {
	Count             int
	ReadState         unifiedresources.ReadState
	Available         bool
	UnavailableReason string
}

const (
	MonitoredSystemUsageUnavailableMonitorState                        = "monitor_state_unavailable"
	MonitoredSystemUsageUnavailableSupplementalInventoryUnsettled      = "supplemental_inventory_unsettled"
	MonitoredSystemUsageUnavailableSupplementalInventoryRebuildPending = "supplemental_inventory_rebuild_pending"
)

// MonitoredSystemUsage returns the canonical monitored-system count only when
// the current unified view is settled enough for settings/support context. When
// supplemental provider-owned sources are still settling, the result fails
// closed with Available=false.
func (m *Monitor) MonitoredSystemUsage() MonitoredSystemUsageSnapshot {
	if m == nil {
		return MonitoredSystemUsageSnapshot{UnavailableReason: MonitoredSystemUsageUnavailableMonitorState}
	}

	readState := m.GetUnifiedReadStateOrSnapshot()
	if readState == nil {
		return MonitoredSystemUsageSnapshot{UnavailableReason: MonitoredSystemUsageUnavailableMonitorState}
	}

	orgID := normalizedMonitorUsageOrgID(m)
	readyAt, settled := m.supplementalInventoryReadyAt(orgID)
	if !settled {
		return MonitoredSystemUsageSnapshot{
			ReadState:         readState,
			UnavailableReason: MonitoredSystemUsageUnavailableSupplementalInventoryUnsettled,
		}
	}
	if !readyAt.IsZero() {
		freshness := m.currentUnifiedResourceFreshness()
		if freshness.IsZero() || freshness.Before(readyAt) {
			return MonitoredSystemUsageSnapshot{
				ReadState:         readState,
				UnavailableReason: MonitoredSystemUsageUnavailableSupplementalInventoryRebuildPending,
			}
		}
	}

	readState = m.readStateWithStandaloneHostContinuity(readState)

	return MonitoredSystemUsageSnapshot{
		Count:     unifiedresources.MonitoredSystemCount(readState),
		ReadState: readState,
		Available: true,
	}
}

func (m *Monitor) readStateWithStandaloneHostContinuity(
	readState unifiedresources.ReadState,
) unifiedresources.ReadState {
	return m.readStateWithStandaloneHostContinuityAt(readState, time.Now().UTC())
}

func (m *Monitor) readStateWithStandaloneHostContinuityAt(
	readState unifiedresources.ReadState,
	now time.Time,
) unifiedresources.ReadState {
	if m == nil || readState == nil {
		return readState
	}

	entries := m.recentStandaloneHostContinuityEntries()
	if len(entries) == 0 {
		return readState
	}

	liveHosts := m.GetLiveHostsSnapshot()
	liveHostIDs := make(map[string]struct{}, len(liveHosts))
	for _, host := range liveHosts {
		if id := strings.TrimSpace(host.ID); id != "" {
			liveHostIDs[id] = struct{}{}
		}
	}

	existingAgentIDs := make(map[string]struct{}, len(readState.Hosts()))
	for _, host := range readState.Hosts() {
		if host == nil {
			continue
		}
		if agentID := strings.TrimSpace(host.AgentID()); agentID != "" {
			existingAgentIDs[agentID] = struct{}{}
		}
		if resourceID := strings.TrimSpace(host.ID()); resourceID != "" {
			existingAgentIDs[resourceID] = struct{}{}
		}
	}

	records := make([]unifiedresources.IngestRecord, 0, len(entries))
	for _, entry := range entries {
		hostID := strings.TrimSpace(entry.HostID)
		if hostID == "" {
			continue
		}
		if _, ok := liveHostIDs[hostID]; ok {
			continue
		}
		if _, ok := existingAgentIDs[hostID]; ok {
			continue
		}
		// A continuity-only row has no live report in this monitor generation.
		// Preserve its last-known identity so the operator can inspect or remove
		// it, but never present persisted telemetry as a current online sighting.
		host := hostFromContinuityEntry(entry)
		host.Status = "offline"
		waiting := hostAgentReportCurrent(entry.LastSeen, entry.IntervalSeconds, now)
		if waiting {
			host.Status = "pending"
		}
		record := unifiedresources.HostIngestRecord(host)
		record.Resource.Metrics = nil
		if waiting {
			// The saved receipt bounds the wait, but is not a delivery in this
			// monitor generation. Keep its timestamp without claiming freshness.
			record.Resource.SourceStatus = map[unifiedresources.DataSource]unifiedresources.SourceStatus{
				unifiedresources.SourceAgent: {Status: "pending", LastSeen: entry.LastSeen},
			}
		}
		records = append(records, record)
	}
	if len(records) == 0 {
		return readState
	}
	return unifiedresources.ReadStateWithHostContinuity(readState, records)
}

func normalizedMonitorUsageOrgID(m *Monitor) string {
	if m == nil {
		return "default"
	}
	orgID := strings.TrimSpace(m.GetOrgID())
	if orgID == "" {
		return "default"
	}
	return orgID
}

func (m *Monitor) supplementalInventoryReadyAt(orgID string) (time.Time, bool) {
	providers := m.supplementalProviderSnapshot()
	if len(providers) == 0 {
		return time.Time{}, true
	}

	var watermark time.Time
	for _, provider := range providers {
		if provider == nil {
			continue
		}

		readinessProvider, hasReadiness := provider.(MonitorSupplementalInventoryReadinessProvider)
		if !hasReadiness {
			if len(supplementalProviderOwnedSourcesForOrg(provider, orgID)) > 0 {
				return time.Time{}, false
			}
			continue
		}

		readyAt, settled := readinessProvider.SupplementalInventoryReadyAt(m, orgID)
		if !settled {
			return time.Time{}, false
		}
		if readyAt.After(watermark) {
			watermark = readyAt
		}
	}

	return watermark, true
}

func supplementalProviderOwnedSourcesForOrg(
	provider MonitorSupplementalRecordsProvider,
	orgID string,
) []unifiedresources.DataSource {
	if provider == nil {
		return nil
	}

	var sources []unifiedresources.DataSource
	if tenantOwner, ok := provider.(interface {
		SnapshotOwnedSourcesForOrg(string) []unifiedresources.DataSource
	}); ok {
		sources = tenantOwner.SnapshotOwnedSourcesForOrg(orgID)
	} else if owner, ok := provider.(interface {
		SnapshotOwnedSources() []unifiedresources.DataSource
	}); ok {
		sources = owner.SnapshotOwnedSources()
	}

	if len(sources) == 0 {
		return nil
	}

	out := make([]unifiedresources.DataSource, 0, len(sources))
	for _, source := range sources {
		normalized := unifiedresources.DataSource(strings.ToLower(strings.TrimSpace(string(source))))
		if normalized == "" {
			continue
		}
		out = append(out, normalized)
	}
	return out
}