Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

fix(mock): re-sight shared storage and Ceph on the PVE cadence

rcourtman63ae0eb6f8d4
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index fdcdcf1aac..9da4bf99c3 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -3783,6 +3783,12 @@ service-history reads plus denial/recovery without fabricated samples.
not the per-node shape the PVE API returns: a shared PBS storage is one
cluster-scoped row listing its reporting nodes (issue 1049), never one row
per node with a different usage figure on each.
+ Cluster-scoped PVE inventory that no single node owns (shared storage under
+ a `cluster` or `shared` owner, and Ceph clusters) is sighted once per PVE
+ cohort rotation while any reporting node is online, the cadence
+ `mock.ProxmoxRefreshInterval` reports, because the node-keyed cohort
+ selection never matches its owner; real PVE polls stamp these rows on every
+ poll.
Demo Docker container names are unique per host, as Docker itself enforces:
a container profile reused for a second round on one host takes a numbered
suffix instead of producing two rows with one name.
diff --git a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md
index 5cdb2e7699..8c59a9f97c 100644
--- a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md
+++ b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md
@@ -818,6 +818,9 @@ A slower mock tick rotates the PVE estate through a divisor of those cohorts
that keeps every node sighted within thirty seconds, down to every node per
tick, so per-second churn never exceeds the two-second default while running
guests stay inside the Proxmox freshness threshold.
+Shared storage and Ceph clusters, which no node owns, are re-sighted only on
+the first cohort of each PVE rotation, so keeping them fresh adds a handful of
+rows to one tick per rotation rather than to every tick.
Mock replication timing is advanced in that same full-rotation refresh, and
only a job whose schedule phase has passed is rewritten, so keeping demo
replication on schedule adds no per-tick change for jobs that did not run.
diff --git a/internal/mock/fixture_graph.go b/internal/mock/fixture_graph.go
index 9cfd529433..1f621d7689 100644
--- a/internal/mock/fixture_graph.go
+++ b/internal/mock/fixture_graph.go
@@ -126,6 +126,10 @@ func (g *FixtureGraph) UpdateMetricCohort(
includeSupplemental: includeSupplemental,
uptimeStep: currentMockUpdateStepInt64() * int64(proxmoxCohorts),
supplementalUptimeStep: currentMockUpdateStepInt64() * supplementalTicks,
+ // Shared storage and Ceph belong to no single node, so they are
+ // sighted once per PVE rotation, the cadence ProxmoxRefreshInterval
+ // reports and the mock Proxmox stale threshold derives from.
+ includeProxmoxClusterScope: proxmoxCohortIndex == 0,
})
if includeSupplemental {
rebaseMockReplicationJobs(g.State.ReplicationJobs, now)
diff --git a/internal/mock/generator.go b/internal/mock/generator.go
index d33645216b..2b86ec4e56 100644
--- a/internal/mock/generator.go
+++ b/internal/mock/generator.go
@@ -6385,6 +6385,10 @@ type fixtureMetricSelection struct {
// whose refresh cadence can differ from the node cohort's rotation. Zero
// falls back to uptimeStep.
supplementalUptimeStep int64
+ // includeProxmoxClusterScope selects PVE inventory that no single node
+ // owns: shared storage merged under a cluster-level owner ("cluster",
+ // "shared") and Ceph clusters. A nil proxmoxNodeNames always includes it.
+ includeProxmoxClusterScope bool
}
func (selection fixtureMetricSelection) includesProxmoxNode(nodeName string) bool {
@@ -6395,6 +6399,10 @@ func (selection fixtureMetricSelection) includesProxmoxNode(nodeName string) boo
return ok
}
+func (selection fixtureMetricSelection) includesProxmoxClusterScope() bool {
+ return selection.proxmoxNodeNames == nil || selection.includeProxmoxClusterScope
+}
+
func updateFixtureStateMetricsSelectedAt(
data *models.StateSnapshot,
config MockConfig,
@@ -6478,8 +6486,11 @@ func updateFixtureStateMetricsSelectedAt(
// stamp is backdated so the UI shows a stale sighting, not "never".
staleSighting := refreshNow.Add(-10 * time.Minute)
offlineNodes := make(map[string]struct{})
+ proxmoxNodes := make(map[string]struct{}, len(data.Nodes))
+ onlineInstances := make(map[string]struct{})
for i := range data.Nodes {
node := &data.Nodes[i]
+ proxmoxNodes[node.Name] = struct{}{}
if node.Status != "online" || node.ConnectionHealth == "offline" || node.Uptime <= 0 {
offlineNodes[node.Name] = struct{}{}
if node.LastSeen.IsZero() {
@@ -6487,11 +6498,41 @@ func updateFixtureStateMetricsSelectedAt(
}
continue
}
+ onlineInstances[node.Instance] = struct{}{}
if !selection.includesProxmoxNode(node.Name) {
continue
}
node.LastSeen = refreshNow
}
+ // Cluster-scoped inventory arrives on every PVE poll through whichever
+ // member node answers, so it is sighted on the cluster-scope cadence
+ // while any of its reporting nodes (or, when none are listed, any node
+ // of its instance) is online. Cohort selection by node name alone never
+ // matches its "cluster"/"shared" owner, which left it unsighted.
+ clusterScoped := func(owner string) bool {
+ _, isNode := proxmoxNodes[owner]
+ return !isNode
+ }
+ clusterReported := func(instance string, reporters []string) bool {
+ if len(reporters) == 0 {
+ _, online := onlineInstances[instance]
+ return online
+ }
+ for _, name := range reporters {
+ if _, offline := offlineNodes[name]; !offline {
+ if _, known := proxmoxNodes[name]; known {
+ return true
+ }
+ }
+ }
+ return false
+ }
+ storageSelected := func(storage *models.Storage) bool {
+ if clusterScoped(storage.Node) {
+ return selection.includesProxmoxClusterScope()
+ }
+ return selection.includesProxmoxNode(storage.Node)
+ }
stampGuestSighting := func(nodeName string, lastSeen *time.Time) {
if !selection.includesProxmoxNode(nodeName) {
return
@@ -6524,7 +6565,29 @@ func updateFixtureStateMetricsSelectedAt(
stampGuestMemoryObservation(data.Containers[i].Node, data.Containers[i].Status, &data.Containers[i].Memory)
}
for i := range data.Storage {
- stampGuestSighting(data.Storage[i].Node, &data.Storage[i].LastSeen)
+ storage := &data.Storage[i]
+ if !clusterScoped(storage.Node) {
+ stampGuestSighting(storage.Node, &storage.LastSeen)
+ continue
+ }
+ if !selection.includesProxmoxClusterScope() {
+ continue
+ }
+ if clusterReported(storage.Instance, storage.Nodes) {
+ storage.LastSeen = refreshNow
+ } else if storage.LastSeen.IsZero() {
+ storage.LastSeen = staleSighting
+ }
+ }
+ if selection.includesProxmoxClusterScope() {
+ for i := range data.CephClusters {
+ cluster := &data.CephClusters[i]
+ if clusterReported(cluster.Instance, nil) {
+ cluster.LastUpdated = refreshNow
+ } else if cluster.LastUpdated.IsZero() {
+ cluster.LastUpdated = staleSighting
+ }
+ }
}
if !config.RandomMetrics {
@@ -6585,7 +6648,7 @@ func updateFixtureStateMetricsSelectedAt(
for i := range data.Storage {
storage := &data.Storage[i]
- if !selection.includesProxmoxNode(storage.Node) || storage.ID == "" || storage.Total <= 0 || storage.Status != "available" {
+ if !storageSelected(storage) || storage.ID == "" || storage.Total <= 0 || storage.Status != "available" {
continue
}
ApplyStorageUsage(storage, SampleMetric("storage", storage.ID, "usage", refreshNow))
diff --git a/internal/mock/platform_fixtures_test.go b/internal/mock/platform_fixtures_test.go
index ccde16a834..6304ebc055 100644
--- a/internal/mock/platform_fixtures_test.go
+++ b/internal/mock/platform_fixtures_test.go
@@ -7,6 +7,7 @@ import (
"go/ast"
"go/parser"
"go/token"
+ "maps"
"math"
"os"
"path/filepath"
@@ -1159,6 +1160,77 @@ func TestFixtureGraphSlowTicksKeepProxmoxGuestSightingsFresh(t *testing.T) {
}
}
+// Shared storage (ceph-rbd, cephfs-data, the cluster PBS stores) and the
+// Ceph cluster belong to no single PVE node, so the node-keyed cohort
+// selection never re-sighted them: they kept their startup stamp and read as
+// stale Proxmox sources on every mock tick rate, including the default.
+func TestFixtureGraphSlowTicksKeepProxmoxClusterSightingsFresh(t *testing.T) {
+ t.Cleanup(func() { setMockUpdateInterval(DefaultConfig.UpdateInterval) })
+
+ for _, interval := range []time.Duration{2 * time.Second, 7 * time.Second, 15 * time.Second, 60 * time.Second} {
+ t.Run(interval.String(), func(t *testing.T) {
+ cfg := DefaultConfig
+ cfg.NodeCount = 8
+ cfg.VMsPerNode = 6
+ cfg.LXCsPerNode = 4
+ cfg.DockerHostCount = 0
+ cfg.GenericHostCount = 0
+ cfg.K8sClusterCount = 0
+ cfg.UpdateInterval = interval
+
+ base := time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC)
+ graph := buildFixtureGraph(cfg, base)
+ nodeNames := make(map[string]struct{}, len(graph.State.Nodes))
+ for _, node := range graph.State.Nodes {
+ nodeNames[node.Name] = struct{}{}
+ }
+ var shared []string
+ for _, storage := range graph.State.Storage {
+ if _, owned := nodeNames[storage.Node]; !owned && storage.Status == "available" {
+ shared = append(shared, storage.ID)
+ }
+ }
+ if !slices.ContainsFunc(shared, func(id string) bool { return strings.HasSuffix(id, "-ceph-rbd") }) {
+ t.Fatalf("fixture estate has no available shared ceph-rbd storage: %v", shared)
+ }
+ if len(graph.State.CephClusters) == 0 {
+ t.Fatal("fixture estate has no Ceph cluster")
+ }
+
+ bound := proxmoxSightingBudget
+ if interval > bound {
+ bound = interval
+ }
+ // Measure just before each tick lands, when sightings are oldest,
+ // and keep each resource's worst age over two full rotations.
+ oldest := make(map[string]time.Duration)
+ observe := func(key string, age time.Duration) {
+ if age > oldest[key] {
+ oldest[key] = age
+ }
+ }
+ for tick := 1; tick <= 2*mockMetricCohortCount; tick++ {
+ now := base.Add(time.Duration(tick) * interval)
+ for _, storage := range graph.State.Storage {
+ if slices.Contains(shared, storage.ID) {
+ observe("shared storage "+storage.ID+" (node "+storage.Node+")", now.Sub(storage.LastSeen))
+ }
+ }
+ for _, cluster := range graph.State.CephClusters {
+ observe("Ceph cluster "+cluster.ID, now.Sub(cluster.LastUpdated))
+ }
+ graph.UpdateMetricCohort(cfg, now, (tick-1)%mockMetricCohortCount, mockMetricCohortCount)
+ }
+ keys := slices.Sorted(maps.Keys(oldest))
+ for _, key := range keys {
+ if oldest[key] > bound {
+ t.Errorf("%s last sighted up to %s ago, want within %s", key, oldest[key], bound)
+ }
+ }
+ })
+ }
+}
+
func TestFixtureGraphMetricCohortProducesSparseUnifiedResourceChanges(t *testing.T) {
cfg := DefaultConfig
cfg.NodeCount = 50