Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
fix(mock): re-sight shared storage and Ceph on the PVE cadence
rcourtman63ae0eb6f8d4
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex fdcdcf1aac..9da4bf99c3 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -3783,6 +3783,12 @@ service-history reads plus denial/recovery without fabricated samples.not the per-node shape the PVE API returns: a shared PBS storage is onecluster-scoped row listing its reporting nodes (issue 1049), never one rowper node with a different usage figure on each.+ Cluster-scoped PVE inventory that no single node owns (shared storage under+ a `cluster` or `shared` owner, and Ceph clusters) is sighted once per PVE+ cohort rotation while any reporting node is online, the cadence+ `mock.ProxmoxRefreshInterval` reports, because the node-keyed cohort+ selection never matches its owner; real PVE polls stamp these rows on every+ poll.Demo Docker container names are unique per host, as Docker itself enforces:a container profile reused for a second round on one host takes a numberedsuffix instead of producing two rows with one name.diff --git a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md b/docs/release-control/v6/internal/subsystems/performance-and-scalability.mdindex 5cdb2e7699..8c59a9f97c 100644--- a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md+++ b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md@@ -818,6 +818,9 @@ A slower mock tick rotates the PVE estate through a divisor of those cohortsthat keeps every node sighted within thirty seconds, down to every node pertick, so per-second churn never exceeds the two-second default while runningguests stay inside the Proxmox freshness threshold.+Shared storage and Ceph clusters, which no node owns, are re-sighted only on+the first cohort of each PVE rotation, so keeping them fresh adds a handful of+rows to one tick per rotation rather than to every tick.Mock replication timing is advanced in that same full-rotation refresh, andonly a job whose schedule phase has passed is rewritten, so keeping demoreplication on schedule adds no per-tick change for jobs that did not run.diff --git a/internal/mock/fixture_graph.go b/internal/mock/fixture_graph.goindex 9cfd529433..1f621d7689 100644--- a/internal/mock/fixture_graph.go+++ b/internal/mock/fixture_graph.go@@ -126,6 +126,10 @@ func (g *FixtureGraph) UpdateMetricCohort(includeSupplemental: includeSupplemental,uptimeStep: currentMockUpdateStepInt64() * int64(proxmoxCohorts),supplementalUptimeStep: currentMockUpdateStepInt64() * supplementalTicks,+ // Shared storage and Ceph belong to no single node, so they are+ // sighted once per PVE rotation, the cadence ProxmoxRefreshInterval+ // reports and the mock Proxmox stale threshold derives from.+ includeProxmoxClusterScope: proxmoxCohortIndex == 0,})if includeSupplemental {rebaseMockReplicationJobs(g.State.ReplicationJobs, now)diff --git a/internal/mock/generator.go b/internal/mock/generator.goindex d33645216b..2b86ec4e56 100644--- a/internal/mock/generator.go+++ b/internal/mock/generator.go@@ -6385,6 +6385,10 @@ type fixtureMetricSelection struct {// whose refresh cadence can differ from the node cohort's rotation. Zero// falls back to uptimeStep.supplementalUptimeStep int64+ // includeProxmoxClusterScope selects PVE inventory that no single node+ // owns: shared storage merged under a cluster-level owner ("cluster",+ // "shared") and Ceph clusters. A nil proxmoxNodeNames always includes it.+ includeProxmoxClusterScope bool}func (selection fixtureMetricSelection) includesProxmoxNode(nodeName string) bool {@@ -6395,6 +6399,10 @@ func (selection fixtureMetricSelection) includesProxmoxNode(nodeName string) booreturn ok}+func (selection fixtureMetricSelection) includesProxmoxClusterScope() bool {+ return selection.proxmoxNodeNames == nil || selection.includeProxmoxClusterScope+}+func updateFixtureStateMetricsSelectedAt(data *models.StateSnapshot,config MockConfig,@@ -6478,8 +6486,11 @@ func updateFixtureStateMetricsSelectedAt(// stamp is backdated so the UI shows a stale sighting, not "never".staleSighting := refreshNow.Add(-10 * time.Minute)offlineNodes := make(map[string]struct{})+ proxmoxNodes := make(map[string]struct{}, len(data.Nodes))+ onlineInstances := make(map[string]struct{})for i := range data.Nodes {node := &data.Nodes[i]+ proxmoxNodes[node.Name] = struct{}{}if node.Status != "online" || node.ConnectionHealth == "offline" || node.Uptime <= 0 {offlineNodes[node.Name] = struct{}{}if node.LastSeen.IsZero() {@@ -6487,11 +6498,41 @@ func updateFixtureStateMetricsSelectedAt(}continue}+ onlineInstances[node.Instance] = struct{}{}if !selection.includesProxmoxNode(node.Name) {continue}node.LastSeen = refreshNow}+ // Cluster-scoped inventory arrives on every PVE poll through whichever+ // member node answers, so it is sighted on the cluster-scope cadence+ // while any of its reporting nodes (or, when none are listed, any node+ // of its instance) is online. Cohort selection by node name alone never+ // matches its "cluster"/"shared" owner, which left it unsighted.+ clusterScoped := func(owner string) bool {+ _, isNode := proxmoxNodes[owner]+ return !isNode+ }+ clusterReported := func(instance string, reporters []string) bool {+ if len(reporters) == 0 {+ _, online := onlineInstances[instance]+ return online+ }+ for _, name := range reporters {+ if _, offline := offlineNodes[name]; !offline {+ if _, known := proxmoxNodes[name]; known {+ return true+ }+ }+ }+ return false+ }+ storageSelected := func(storage *models.Storage) bool {+ if clusterScoped(storage.Node) {+ return selection.includesProxmoxClusterScope()+ }+ return selection.includesProxmoxNode(storage.Node)+ }stampGuestSighting := func(nodeName string, lastSeen *time.Time) {if !selection.includesProxmoxNode(nodeName) {return@@ -6524,7 +6565,29 @@ func updateFixtureStateMetricsSelectedAt(stampGuestMemoryObservation(data.Containers[i].Node, data.Containers[i].Status, &data.Containers[i].Memory)}for i := range data.Storage {- stampGuestSighting(data.Storage[i].Node, &data.Storage[i].LastSeen)+ storage := &data.Storage[i]+ if !clusterScoped(storage.Node) {+ stampGuestSighting(storage.Node, &storage.LastSeen)+ continue+ }+ if !selection.includesProxmoxClusterScope() {+ continue+ }+ if clusterReported(storage.Instance, storage.Nodes) {+ storage.LastSeen = refreshNow+ } else if storage.LastSeen.IsZero() {+ storage.LastSeen = staleSighting+ }+ }+ if selection.includesProxmoxClusterScope() {+ for i := range data.CephClusters {+ cluster := &data.CephClusters[i]+ if clusterReported(cluster.Instance, nil) {+ cluster.LastUpdated = refreshNow+ } else if cluster.LastUpdated.IsZero() {+ cluster.LastUpdated = staleSighting+ }+ }}if !config.RandomMetrics {@@ -6585,7 +6648,7 @@ func updateFixtureStateMetricsSelectedAt(for i := range data.Storage {storage := &data.Storage[i]- if !selection.includesProxmoxNode(storage.Node) || storage.ID == "" || storage.Total <= 0 || storage.Status != "available" {+ if !storageSelected(storage) || storage.ID == "" || storage.Total <= 0 || storage.Status != "available" {continue}ApplyStorageUsage(storage, SampleMetric("storage", storage.ID, "usage", refreshNow))diff --git a/internal/mock/platform_fixtures_test.go b/internal/mock/platform_fixtures_test.goindex ccde16a834..6304ebc055 100644--- a/internal/mock/platform_fixtures_test.go+++ b/internal/mock/platform_fixtures_test.go@@ -7,6 +7,7 @@ import ("go/ast""go/parser""go/token"+ "maps""math""os""path/filepath"@@ -1159,6 +1160,77 @@ func TestFixtureGraphSlowTicksKeepProxmoxGuestSightingsFresh(t *testing.T) {}}+// Shared storage (ceph-rbd, cephfs-data, the cluster PBS stores) and the+// Ceph cluster belong to no single PVE node, so the node-keyed cohort+// selection never re-sighted them: they kept their startup stamp and read as+// stale Proxmox sources on every mock tick rate, including the default.+func TestFixtureGraphSlowTicksKeepProxmoxClusterSightingsFresh(t *testing.T) {+ t.Cleanup(func() { setMockUpdateInterval(DefaultConfig.UpdateInterval) })++ for _, interval := range []time.Duration{2 * time.Second, 7 * time.Second, 15 * time.Second, 60 * time.Second} {+ t.Run(interval.String(), func(t *testing.T) {+ cfg := DefaultConfig+ cfg.NodeCount = 8+ cfg.VMsPerNode = 6+ cfg.LXCsPerNode = 4+ cfg.DockerHostCount = 0+ cfg.GenericHostCount = 0+ cfg.K8sClusterCount = 0+ cfg.UpdateInterval = interval++ base := time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC)+ graph := buildFixtureGraph(cfg, base)+ nodeNames := make(map[string]struct{}, len(graph.State.Nodes))+ for _, node := range graph.State.Nodes {+ nodeNames[node.Name] = struct{}{}+ }+ var shared []string+ for _, storage := range graph.State.Storage {+ if _, owned := nodeNames[storage.Node]; !owned && storage.Status == "available" {+ shared = append(shared, storage.ID)+ }+ }+ if !slices.ContainsFunc(shared, func(id string) bool { return strings.HasSuffix(id, "-ceph-rbd") }) {+ t.Fatalf("fixture estate has no available shared ceph-rbd storage: %v", shared)+ }+ if len(graph.State.CephClusters) == 0 {+ t.Fatal("fixture estate has no Ceph cluster")+ }++ bound := proxmoxSightingBudget+ if interval > bound {+ bound = interval+ }+ // Measure just before each tick lands, when sightings are oldest,+ // and keep each resource's worst age over two full rotations.+ oldest := make(map[string]time.Duration)+ observe := func(key string, age time.Duration) {+ if age > oldest[key] {+ oldest[key] = age+ }+ }+ for tick := 1; tick <= 2*mockMetricCohortCount; tick++ {+ now := base.Add(time.Duration(tick) * interval)+ for _, storage := range graph.State.Storage {+ if slices.Contains(shared, storage.ID) {+ observe("shared storage "+storage.ID+" (node "+storage.Node+")", now.Sub(storage.LastSeen))+ }+ }+ for _, cluster := range graph.State.CephClusters {+ observe("Ceph cluster "+cluster.ID, now.Sub(cluster.LastUpdated))+ }+ graph.UpdateMetricCohort(cfg, now, (tick-1)%mockMetricCohortCount, mockMetricCohortCount)+ }+ keys := slices.Sorted(maps.Keys(oldest))+ for _, key := range keys {+ if oldest[key] > bound {+ t.Errorf("%s last sighted up to %s ago, want within %s", key, oldest[key], bound)+ }+ }+ })+ }+}+func TestFixtureGraphMetricCohortProducesSparseUnifiedResourceChanges(t *testing.T) {cfg := DefaultConfigcfg.NodeCount = 50