Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

15.2 KB
Raw Blame
package mock

import (
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/ai/memory"
	"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/internal/truenas"
	"github.com/rcourtman/pulse-go-rewrite/internal/vmware"
)

// FixtureGraph is the canonical mock runtime owner for snapshot-backed and
// provider-backed fixtures. All mock projections should derive from this graph
// rather than mixing independent snapshot and provider helpers.
type FixtureGraph struct {
	State                models.StateSnapshot
	AlertHistory         []models.Alert
	AlertIncidents       []*memory.Incident
	PlatformFixtures     PlatformFixtures
	AvailabilityFixtures []AvailabilityFixture
	ActionFixtures       []ActionFixture
}

func emptyFixtureGraph() FixtureGraph {
	return FixtureGraph{
		State: models.EmptyStateSnapshot(),
	}
}

func buildFixtureGraph(cfg MockConfig, now time.Time) FixtureGraph {
	setMockUpdateInterval(cfg.UpdateInterval)
	graph := FixtureGraph{
		State:                buildFixtureState(cfg),
		PlatformFixtures:     defaultPlatformFixtures(),
		AvailabilityFixtures: defaultAvailabilityFixtures(now),
	}
	applyDemoScenarioGraph(&graph, cfg, now)
	syncMetricFixtureRegistriesFromGraph(graph)
	graph.UpdateMetrics(cfg, now)
	graph.AlertHistory = buildAlertHistory(graph.State.Nodes, graph.State.VMs, graph.State.Containers)
	graph.AlertHistory, graph.AlertIncidents = buildAlertIncidentFixtures(graph.AlertHistory, now)
	resources, _ := graph.UnifiedResourceSnapshot()
	graph.ActionFixtures = buildActionFixtures(resources, now)
	syncMetricFixtureRegistriesFromGraph(graph)
	return graph
}

func cloneFixtureGraph(in FixtureGraph) FixtureGraph {
	return FixtureGraph{
		State:                cloneState(in.State),
		AlertHistory:         cloneMockAlerts(in.AlertHistory),
		AlertIncidents:       cloneMockIncidents(in.AlertIncidents),
		PlatformFixtures:     clonePlatformFixtures(in.PlatformFixtures),
		AvailabilityFixtures: cloneAvailabilityFixtures(in.AvailabilityFixtures),
		ActionFixtures:       cloneActionFixtures(in.ActionFixtures),
	}
}

func (g *FixtureGraph) UpdateMetrics(cfg MockConfig, now time.Time) {
	if g == nil {
		return
	}

	setMockUpdateInterval(cfg.UpdateInterval)
	applyDemoScenarioGraph(g, cfg, now)
	syncMetricFixtureRegistriesFromGraph(*g)
	updateFixtureStateMetricsAt(&g.State, cfg, now)
	rebaseMockReplicationJobs(g.State.ReplicationJobs, now)
	g.PlatformFixtures = rebasePlatformFixtures(g.PlatformFixtures, now)
	g.AvailabilityFixtures = rebaseAvailabilityFixtures(g.AvailabilityFixtures, now)
	applyDemoScenarioGraph(g, cfg, now)
	syncMetricFixtureRegistriesFromGraph(*g)
}

// UpdateMetricCohort advances one bounded slice of the demo estate. The full
// UpdateMetrics path remains the fixture-construction and explicit refresh
// boundary; the long-running mock sampler uses cohorts so a large public demo
// does not make every resource appear changed on every two-second tick.
func (g *FixtureGraph) UpdateMetricCohort(
	cfg MockConfig,
	now time.Time,
	cohortIndex int,
	cohortCount int,
) {
	if g == nil {
		return
	}
	if cohortCount <= 0 {
		cohortCount = 1
	}
	cohortIndex %= cohortCount
	if cohortIndex < 0 {
		cohortIndex += cohortCount
	}

	setMockUpdateInterval(cfg.UpdateInterval)
	// PVE sightings rotate through a divisor of cohortCount, so a slow tick
	// still sights every node and its guests inside the Proxmox freshness
	// budget instead of once per full (tick x cohortCount) rotation.
	proxmoxCohorts := proxmoxCohortCount(currentMockUpdateInterval(), cohortCount)
	proxmoxCohortIndex := cohortIndex % proxmoxCohorts
	selectedNodes := make(map[string]struct{}, (len(g.State.Nodes)+proxmoxCohorts-1)/proxmoxCohorts)
	for index := range g.State.Nodes {
		if index%proxmoxCohorts != proxmoxCohortIndex {
			continue
		}
		selectedNodes[g.State.Nodes[index].Name] = struct{}{}
	}

	// Provider-backed demo fixtures are much smaller than the PVE estate and
	// normally refresh once per full cohort rotation, keeping their timestamps
	// live without forcing every provider resource through every socket delta.
	// When a slow tick interval stretches the rotation past the registry's
	// freshness budget, they refresh every tick instead — otherwise every
	// TrueNAS/VMware/availability row spends most of each rotation flagged as
	// a stale source (all-Attention badge noise on the public demo).
	supplementalEveryTick := supplementalRefreshEveryTick(currentMockUpdateInterval(), cohortCount)
	includeSupplemental := cohortIndex == 0 || supplementalEveryTick
	supplementalTicks := int64(cohortCount)
	if supplementalEveryTick {
		supplementalTicks = 1
	}
	updateFixtureStateMetricsSelectedAt(&g.State, cfg, now, fixtureMetricSelection{
		proxmoxNodeNames:       selectedNodes,
		includeSupplemental:    includeSupplemental,
		uptimeStep:             currentMockUpdateStepInt64() * int64(proxmoxCohorts),
		supplementalUptimeStep: currentMockUpdateStepInt64() * supplementalTicks,
		// Shared storage and Ceph belong to no single node, so they are
		// sighted once per PVE rotation, the cadence ProxmoxRefreshInterval
		// reports and the mock Proxmox stale threshold derives from.
		includeProxmoxClusterScope: proxmoxCohortIndex == 0,
	})
	if includeSupplemental {
		rebaseMockReplicationJobs(g.State.ReplicationJobs, now)
		g.PlatformFixtures = rebasePlatformFixtures(g.PlatformFixtures, now)
		g.AvailabilityFixtures = rebaseAvailabilityFixtures(g.AvailabilityFixtures, now)
	}
	syncMetricFixtureRegistriesFromGraph(*g)
}

func (g *FixtureGraph) UpdateAlertSnapshots(active []alerts.Alert, resolved []models.ResolvedAlert) {
	if g == nil {
		return
	}

	converted := make([]models.Alert, 0, len(active))
	for _, alert := range active {
		converted = append(converted, models.Alert{
			ID:           alert.ID,
			Type:         alert.Type,
			Level:        string(alert.Level),
			ResourceID:   alert.ResourceID,
			ResourceName: alert.ResourceName,
			Node:         alert.Node,
			Instance:     alert.Instance,
			Message:      alert.Message,
			Summary:      alert.Summary,
			Value:        alert.Value,
			Threshold:    alert.Threshold,
			StartTime:    alert.StartTime,
			LastSeen:     cloneMockTime(alert.LastSeen),
			Acknowledged: alert.Acknowledged,
			// GetActiveAlerts returns deep clones, so the map is already private.
			Metadata: alert.Metadata,
		})
	}

	g.State.ActiveAlerts = converted
	g.State.RecentlyResolved = append([]models.ResolvedAlert(nil), resolved...)
}

func CurrentFixtureGraph() FixtureGraph {
	graph, _ := CurrentFixtureGraphWithRevision()
	return graph
}

// CurrentFixtureGraphWithRevision returns one coherent fixture snapshot and
// its structural revision so downstream caches cannot pair an old graph with
// a newer invalidation token.
func CurrentFixtureGraphWithRevision() (FixtureGraph, uint64) {
	dataMu.RLock()
	defer dataMu.RUnlock()

	if !enabled.Load() {
		return emptyFixtureGraph(), fixtureRevision.Load()
	}
	return cloneFixtureGraph(mockGraph), fixtureRevision.Load()
}

func currentOrDefaultPlatformFixtures() PlatformFixtures {
	if !IsMockEnabled() {
		return defaultPlatformFixtures()
	}

	return CurrentFixtureGraph().PlatformFixtures
}

func clonePlatformFixtures(in PlatformFixtures) PlatformFixtures {
	return PlatformFixtures{
		TrueNAS: cloneTrueNASFixtureSnapshot(in.TrueNAS),
		VMware:  cloneVMwareInventorySnapshot(in.VMware),
	}
}

func cloneTrueNASFixtureSnapshot(in truenas.FixtureSnapshot) truenas.FixtureSnapshot {
	out := in
	out.System = cloneTrueNASSystemInfo(in.System)
	out.Pools = cloneTrueNASPools(in.Pools)
	out.Datasets = append([]truenas.Dataset(nil), in.Datasets...)
	out.Disks = append([]truenas.Disk(nil), in.Disks...)
	out.Alerts = append([]truenas.Alert(nil), in.Alerts...)
	out.Apps = cloneTrueNASApps(in.Apps)
	out.ZFSSnapshots = cloneTrueNASZFSSnapshots(in.ZFSSnapshots)
	out.ReplicationTasks = cloneTrueNASReplicationTasks(in.ReplicationTasks)
	return out
}

func cloneTrueNASPools(in []truenas.Pool) []truenas.Pool {
	if in == nil {
		return nil
	}

	out := make([]truenas.Pool, len(in))
	for i := range in {
		out[i] = in[i]
		if in[i].Scan != nil {
			scan := *in[i].Scan
			out[i].Scan = &scan
		}
		out[i].VDevs = append([]truenas.PoolVDev(nil), in[i].VDevs...)
		out[i].DiskMembers = append([]truenas.PoolDiskMember(nil), in[i].DiskMembers...)
	}
	return out
}

func cloneTrueNASSystemInfo(in truenas.SystemInfo) truenas.SystemInfo {
	out := in
	if len(in.TemperatureCelsius) > 0 {
		out.TemperatureCelsius = make(map[string]float64, len(in.TemperatureCelsius))
		for key, value := range in.TemperatureCelsius {
			out.TemperatureCelsius[key] = value
		}
	}
	return out
}

func cloneTrueNASApps(in []truenas.App) []truenas.App {
	if in == nil {
		return nil
	}

	out := make([]truenas.App, len(in))
	for i := range in {
		out[i] = in[i]
		out[i].UsedHostIPs = append([]string(nil), in[i].UsedHostIPs...)
		out[i].UsedPorts = cloneTrueNASAppPorts(in[i].UsedPorts)
		out[i].Containers = cloneTrueNASAppContainers(in[i].Containers)
		out[i].Volumes = append([]truenas.AppVolume(nil), in[i].Volumes...)
		out[i].Images = append([]string(nil), in[i].Images...)
		out[i].Networks = cloneTrueNASAppNetworks(in[i].Networks)
		out[i].Stats = cloneTrueNASAppStats(in[i].Stats)
	}

	return out
}

func cloneTrueNASAppPorts(in []truenas.AppPort) []truenas.AppPort {
	if in == nil {
		return nil
	}

	out := make([]truenas.AppPort, len(in))
	for i := range in {
		out[i] = in[i]
		out[i].HostPorts = append([]truenas.AppHostPort(nil), in[i].HostPorts...)
	}
	return out
}

func cloneTrueNASAppContainers(in []truenas.AppContainer) []truenas.AppContainer {
	if in == nil {
		return nil
	}

	out := make([]truenas.AppContainer, len(in))
	for i := range in {
		out[i] = in[i]
		out[i].PortConfig = cloneTrueNASAppPorts(in[i].PortConfig)
		out[i].VolumeMounts = append([]truenas.AppVolume(nil), in[i].VolumeMounts...)
	}
	return out
}

func cloneTrueNASAppNetworks(in []truenas.AppNetwork) []truenas.AppNetwork {
	if in == nil {
		return nil
	}

	out := make([]truenas.AppNetwork, len(in))
	for i := range in {
		out[i] = in[i]
		out[i].Labels = cloneStringMap(in[i].Labels)
	}
	return out
}

func cloneTrueNASAppStats(in *truenas.AppStats) *truenas.AppStats {
	if in == nil {
		return nil
	}

	out := *in
	out.Interfaces = append([]truenas.AppInterfaceStats(nil), in.Interfaces...)
	return &out
}

func cloneTrueNASZFSSnapshots(in []truenas.ZFSSnapshot) []truenas.ZFSSnapshot {
	if in == nil {
		return nil
	}

	out := make([]truenas.ZFSSnapshot, len(in))
	for i := range in {
		out[i] = in[i]
		out[i].CreatedAt = cloneTimePtr(in[i].CreatedAt)
		out[i].UsedBytes = cloneInt64Ptr(in[i].UsedBytes)
		out[i].Referenced = cloneInt64Ptr(in[i].Referenced)
	}
	return out
}

func cloneTrueNASReplicationTasks(in []truenas.ReplicationTask) []truenas.ReplicationTask {
	if in == nil {
		return nil
	}

	out := make([]truenas.ReplicationTask, len(in))
	for i := range in {
		out[i] = in[i]
		out[i].SourceDatasets = append([]string(nil), in[i].SourceDatasets...)
		out[i].LastRun = cloneTimePtr(in[i].LastRun)
	}
	return out
}

func cloneVMwareInventorySnapshot(in vmware.InventorySnapshot) vmware.InventorySnapshot {
	out := in
	out.Hosts = cloneVMwareInventoryHosts(in.Hosts)
	out.VMs = cloneVMwareInventoryVMs(in.VMs)
	out.Datastores = cloneVMwareInventoryDatastores(in.Datastores)
	out.Networks = cloneVMwareInventoryNetworks(in.Networks)
	out.EnrichmentIssues = append([]vmware.InventoryEnrichmentIssue(nil), in.EnrichmentIssues...)
	return out
}

func cloneVMwareInventoryHosts(in []vmware.InventoryHost) []vmware.InventoryHost {
	if in == nil {
		return nil
	}

	out := make([]vmware.InventoryHost, len(in))
	for i := range in {
		out[i] = in[i]
		out[i].DatastoreIDs = append([]string(nil), in[i].DatastoreIDs...)
		out[i].DatastoreNames = append([]string(nil), in[i].DatastoreNames...)
		out[i].TriggeredAlarms = append([]vmware.InventoryAlarm(nil), in[i].TriggeredAlarms...)
		out[i].RecentTasks = append([]vmware.InventoryTask(nil), in[i].RecentTasks...)
		out[i].RecentEvents = append([]vmware.InventoryEvent(nil), in[i].RecentEvents...)
		out[i].Metrics = cloneVMwareInventoryMetrics(in[i].Metrics)
	}

	return out
}

func cloneVMwareInventoryVMs(in []vmware.InventoryVM) []vmware.InventoryVM {
	if in == nil {
		return nil
	}

	out := make([]vmware.InventoryVM, len(in))
	for i := range in {
		out[i] = in[i]
		out[i].DatastoreIDs = append([]string(nil), in[i].DatastoreIDs...)
		out[i].DatastoreNames = append([]string(nil), in[i].DatastoreNames...)
		out[i].GuestIPAddresses = append([]string(nil), in[i].GuestIPAddresses...)
		out[i].TriggeredAlarms = append([]vmware.InventoryAlarm(nil), in[i].TriggeredAlarms...)
		out[i].RecentTasks = append([]vmware.InventoryTask(nil), in[i].RecentTasks...)
		out[i].RecentEvents = append([]vmware.InventoryEvent(nil), in[i].RecentEvents...)
		out[i].Metrics = cloneVMwareInventoryMetrics(in[i].Metrics)
	}

	return out
}

func cloneVMwareInventoryDatastores(in []vmware.InventoryDatastore) []vmware.InventoryDatastore {
	if in == nil {
		return nil
	}

	out := make([]vmware.InventoryDatastore, len(in))
	for i := range in {
		out[i] = in[i]
		out[i].HostIDs = append([]string(nil), in[i].HostIDs...)
		out[i].HostNames = append([]string(nil), in[i].HostNames...)
		out[i].VMIDs = append([]string(nil), in[i].VMIDs...)
		out[i].VMNames = append([]string(nil), in[i].VMNames...)
		out[i].Accessible = cloneBoolPtr(in[i].Accessible)
		out[i].MultipleHostAccess = cloneBoolPtr(in[i].MultipleHostAccess)
		out[i].TriggeredAlarms = append([]vmware.InventoryAlarm(nil), in[i].TriggeredAlarms...)
		out[i].RecentTasks = append([]vmware.InventoryTask(nil), in[i].RecentTasks...)
		out[i].RecentEvents = append([]vmware.InventoryEvent(nil), in[i].RecentEvents...)
	}

	return out
}

func cloneVMwareInventoryNetworks(in []vmware.InventoryNetwork) []vmware.InventoryNetwork {
	if in == nil {
		return nil
	}

	out := make([]vmware.InventoryNetwork, len(in))
	for i := range in {
		out[i] = in[i]
		out[i].HostIDs = append([]string(nil), in[i].HostIDs...)
		out[i].HostNames = append([]string(nil), in[i].HostNames...)
		out[i].VMIDs = append([]string(nil), in[i].VMIDs...)
		out[i].VMNames = append([]string(nil), in[i].VMNames...)
		out[i].TriggeredAlarms = append([]vmware.InventoryAlarm(nil), in[i].TriggeredAlarms...)
		out[i].RecentTasks = append([]vmware.InventoryTask(nil), in[i].RecentTasks...)
		out[i].RecentEvents = append([]vmware.InventoryEvent(nil), in[i].RecentEvents...)
	}

	return out
}

func cloneVMwareInventoryMetrics(in *vmware.InventoryMetrics) *vmware.InventoryMetrics {
	if in == nil {
		return nil
	}

	out := *in
	out.CPUPercent = cloneFloat64Ptr(in.CPUPercent)
	out.MemoryPercent = cloneFloat64Ptr(in.MemoryPercent)
	out.MemoryUsedBytes = cloneInt64Ptr(in.MemoryUsedBytes)
	out.MemoryTotalBytes = cloneInt64Ptr(in.MemoryTotalBytes)
	out.NetInBytesPerSecond = cloneFloat64Ptr(in.NetInBytesPerSecond)
	out.NetOutBytesPerSecond = cloneFloat64Ptr(in.NetOutBytesPerSecond)
	out.DiskReadBytesPerSecond = cloneFloat64Ptr(in.DiskReadBytesPerSecond)
	out.DiskWriteBytesPerSecond = cloneFloat64Ptr(in.DiskWriteBytesPerSecond)
	out.UptimeSeconds = cloneInt64Ptr(in.UptimeSeconds)
	out.DiskUsedBytes = cloneInt64Ptr(in.DiskUsedBytes)
	out.DiskTotalBytes = cloneInt64Ptr(in.DiskTotalBytes)
	out.DiskPercent = cloneFloat64Ptr(in.DiskPercent)
	return &out
}

func cloneFloat64Ptr(n *float64) *float64 {
	if n == nil {
		return nil
	}
	value := *n
	return &value
}