Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

fix(monitoring): publish complete resource wiring once

pulse-triage[bot]da44cec421a1
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index b198549729..18f15e5aa2 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -17,6 +17,40 @@
## Purpose
+### Repeatable resource wiring publishes a complete binding once
+
+Router and tenant initialization applies saved notification policy before the
+first resource publication, then attaches the store and supplemental provider
+batch together. Repeating identical wiring does not rebuild the estate; the
+single setters retain their explicit-refresh behaviour. Explicit nil provider
+withdrawals remain in the router's initializer registry so existing and future
+tenants cannot retain a removed source. An empty saved private-CIDR allowlist
+revokes previous grants; absent/unreadable settings and the automatic public-URL
+fallback retain their existing semantics.
+
+A refresh captures its store/provider generation before collecting records.
+Delayed old collectors are refused before publication; full store writes are
+ordered after collection, without holding monitor or provider locks across
+callbacks. Changed bindings advance the existing alert epoch and join admitted
+old checks without clearing active alerts. Dependency setters, like a mock-mode
+switch, must not change bindings from an admitted alert callback. Consume-once
+source changes are drained only after an admitted publication, outside the
+publication lock. The read path keeps its existing non-blocking/freshness gate.
+
+Native TrueNAS/VMware providers and the demo adapter declare their mode. One
+mode read filters the pair before records, ownership or changes are collected,
+including the interval before the existing post-switch hook replaces them.
+Opposite-mode data cannot become a live/demo mixture; tenant fixture exclusions
+remain. Runtime-check shutdown, cancellation, token/tenant boundaries, resource
+wire shapes, thresholds, retention and guest/TLS safety fences are unchanged.
+
+`resource_wiring_test.go` and `router_resource_wiring_test.go` cover pass counts,
+first-alert policy, allowlist revocation, withdrawals, delayed collection and
+publication, re-entry, admitted-check joins and switch-hook gaps. These are
+bounded synthetic component controls, not whole-source, native startup/RSS,
+notification delivery or installed/reporter acceptance.
+
+
### Guest filesystem guidance honours its caller's logger
Missing-filesystem guidance uses the existing caller context logger, falling
diff --git a/internal/api/router.go b/internal/api/router.go
index fe2766d9e5..4d92680657 100644
--- a/internal/api/router.go
+++ b/internal/api/router.go
@@ -112,6 +112,7 @@ type Router struct {
monitorResourceAdapter *unifiedresources.MonitorAdapter
monitorResourceAdapters map[string]*unifiedresources.MonitorAdapter
monitorAdapterMu sync.Mutex
+ monitorSupplementalMu sync.Mutex // serializes provider updates with complete monitor initialization
monitorSupplementalRecords map[unifiedresources.DataSource]monitoring.MonitorSupplementalRecordsProvider
reportingHandlers *ReportingHandlers
configProfileHandler *ConfigProfileHandler
@@ -1882,11 +1883,15 @@ func (r *Router) configureMonitorDependencies(m *monitoring.Monitor) {
return
}
- if adapter := r.monitorAdapterForMonitor(m); adapter != nil {
- log.Debug().Msg("[Router] Injecting unified resource adapter into monitor")
- m.SetResourceStore(adapter)
- }
+ r.monitorSupplementalMu.Lock()
+ defer r.monitorSupplementalMu.Unlock()
+ r.configureMonitorDependenciesLocked(m)
+}
+// The caller holds monitorSupplementalMu across reading and applying the
+// registry, not just copying the map: a delayed initializer must not reinstall
+// an old provider after a mock switch or withdrawal has completed.
+func (r *Router) configureMonitorDependenciesLocked(m *monitoring.Monitor) {
// Tenant monitors must inherit the persisted instance-wide notification
// settings. System settings are stored globally, so without this a tenant
// monitor created after the allowlist was saved (or after a restart)
@@ -1896,11 +1901,11 @@ func (r *Router) configureMonitorDependencies(m *monitoring.Monitor) {
if r.persistence != nil {
if settings, err := r.persistence.LoadSystemSettings(); err == nil && settings != nil {
if nm := m.GetNotificationManager(); nm != nil {
- if settings.WebhookAllowedPrivateCIDRs != "" {
- if err := nm.UpdateAllowedPrivateCIDRs(settings.WebhookAllowedPrivateCIDRs); err != nil {
- log.Error().Err(err).Msg("Failed to apply webhook allowed private CIDRs to tenant monitor")
- }
+ if err := nm.UpdateAllowedPrivateCIDRs(settings.WebhookAllowedPrivateCIDRs); err != nil {
+ log.Error().Err(err).Msg("Failed to apply webhook allowed private CIDRs to tenant monitor")
}
+ // An empty allowlist revokes old grants. An empty public URL
+ // keeps the notification manager's existing automatic fallback.
if settings.PublicURL != "" {
nm.SetPublicURL(settings.PublicURL)
}
@@ -1910,46 +1915,45 @@ func (r *Router) configureMonitorDependencies(m *monitoring.Monitor) {
}
}
- if len(r.monitorSupplementalRecords) == 0 {
- return
- }
-
- keys := make([]string, 0, len(r.monitorSupplementalRecords))
- for source := range r.monitorSupplementalRecords {
- keys = append(keys, string(source))
- }
- sort.Strings(keys)
-
- for _, key := range keys {
- source := unifiedresources.DataSource(key)
- provider := r.monitorSupplementalRecords[source]
- m.SetSupplementalRecordsProvider(source, provider)
+ // Apply saved notification policy before the first publication, which can
+ // evaluate alerts. The combined boundary avoids one refresh per source.
+ if adapter := r.monitorAdapterForMonitor(m); adapter != nil {
+ m.SetResourceStoreAndProviders(adapter, r.monitorSupplementalRecords)
+ } else {
+ m.SetSupplementalRecordsProviders(r.monitorSupplementalRecords)
}
}
func (r *Router) setMonitorSupplementalRecordsProvider(source unifiedresources.DataSource, provider monitoring.MonitorSupplementalRecordsProvider) {
- if r == nil {
- return
- }
+ r.setMonitorSupplementalRecordsProviders(map[unifiedresources.DataSource]monitoring.MonitorSupplementalRecordsProvider{source: provider})
+}
- normalized := unifiedresources.DataSource(strings.ToLower(strings.TrimSpace(string(source))))
- if normalized == "" {
+func (r *Router) setMonitorSupplementalRecordsProviders(providers map[unifiedresources.DataSource]monitoring.MonitorSupplementalRecordsProvider) {
+ if r == nil {
return
}
-
- if r.monitorSupplementalRecords == nil {
- r.monitorSupplementalRecords = make(map[unifiedresources.DataSource]monitoring.MonitorSupplementalRecordsProvider)
- }
- if provider == nil {
- delete(r.monitorSupplementalRecords, normalized)
- } else {
+ r.monitorSupplementalMu.Lock()
+ usable := false
+ for source, provider := range providers {
+ normalized := unifiedresources.DataSource(strings.ToLower(strings.TrimSpace(string(source))))
+ if normalized == "" {
+ continue
+ }
+ usable = true
+ if r.monitorSupplementalRecords == nil {
+ r.monitorSupplementalRecords = make(map[unifiedresources.DataSource]monitoring.MonitorSupplementalRecordsProvider)
+ }
+ // Retain explicit nil withdrawals in initializer snapshots. Omitting a
+ // removed source would leave it installed in already-running tenants.
r.monitorSupplementalRecords[normalized] = provider
}
-
- if r.monitor != nil {
- r.monitor.SetSupplementalRecordsProvider(normalized, provider)
+ if usable && r.monitor != nil {
+ r.configureMonitorDependenciesLocked(r.monitor)
}
- if r.mtMonitor != nil {
+ r.monitorSupplementalMu.Unlock()
+ if usable && r.mtMonitor != nil {
+ // Do not hold the router lock while taking the tenant-manager lock.
+ // Its callback obtains a fresh, serialized registry for each monitor.
r.mtMonitor.SetMonitorInitializer(r.configureMonitorDependencies)
}
}
@@ -1970,8 +1974,10 @@ func (r *Router) syncPlatformSupplementalProviders(mockEnabled bool) {
r.resourceHandlers.SetSupplementalRecordsProvider(unifiedresources.SourceTrueNAS, trueNASAdapter)
r.resourceHandlers.SetSupplementalRecordsProvider(unifiedresources.SourceVMware, vmwareAdapter)
}
- r.setMonitorSupplementalRecordsProvider(unifiedresources.SourceTrueNAS, trueNASAdapter)
- r.setMonitorSupplementalRecordsProvider(unifiedresources.SourceVMware, vmwareAdapter)
+ r.setMonitorSupplementalRecordsProviders(map[unifiedresources.DataSource]monitoring.MonitorSupplementalRecordsProvider{
+ unifiedresources.SourceTrueNAS: trueNASAdapter,
+ unifiedresources.SourceVMware: vmwareAdapter,
+ })
return
}
@@ -1982,8 +1988,10 @@ func (r *Router) syncPlatformSupplementalProviders(mockEnabled bool) {
r.resourceHandlers.SetSupplementalRecordsProvider(unifiedresources.SourceTrueNAS, r.trueNASPoller)
r.resourceHandlers.SetSupplementalRecordsProvider(unifiedresources.SourceVMware, r.vmwarePoller)
}
- r.setMonitorSupplementalRecordsProvider(unifiedresources.SourceTrueNAS, r.trueNASPoller)
- r.setMonitorSupplementalRecordsProvider(unifiedresources.SourceVMware, r.vmwarePoller)
+ r.setMonitorSupplementalRecordsProviders(map[unifiedresources.DataSource]monitoring.MonitorSupplementalRecordsProvider{
+ unifiedresources.SourceTrueNAS: r.trueNASPoller,
+ unifiedresources.SourceVMware: r.vmwarePoller,
+ })
}
// getTenantMonitor returns the appropriate monitor for the current request's tenant.
@@ -8442,6 +8450,8 @@ func (a mockSupplementalRecordsAdapter) GetCurrentRecordsForOrg(orgID string) []
return mock.SupplementalRecords(a.source)
}
+func (mockSupplementalRecordsAdapter) SupplementalRecordsMockMode() bool { return true }
+
func (a mockSupplementalRecordsAdapter) SupplementalRecords(_ *monitoring.Monitor, orgID string) []unifiedresources.IngestRecord {
return a.GetCurrentRecordsForOrg(orgID)
}
diff --git a/internal/api/router_resource_wiring_test.go b/internal/api/router_resource_wiring_test.go
new file mode 100644
index 0000000000..00245a0676
--- /dev/null
+++ b/internal/api/router_resource_wiring_test.go
@@ -0,0 +1,295 @@
+package api
+
+import (
+ "github.com/rcourtman/pulse-go-rewrite/internal/alerts"
+ "github.com/rcourtman/pulse-go-rewrite/internal/config"
+ "github.com/rcourtman/pulse-go-rewrite/internal/models"
+ "github.com/rcourtman/pulse-go-rewrite/internal/monitoring"
+ "github.com/rcourtman/pulse-go-rewrite/internal/truenas"
+ "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
+ "github.com/rcourtman/pulse-go-rewrite/internal/vmware"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+)
+
+type countingSupplementalRecords struct {
+ reads atomic.Int32
+}
+
+func (c *countingSupplementalRecords) SupplementalRecords(*monitoring.Monitor, string) []unifiedresources.IngestRecord {
+ c.reads.Add(1)
+ return nil
+}
+
+func newWiringTestMonitor(t *testing.T) *monitoring.Monitor {
+ t.Helper()
+ monitor, err := monitoring.New(&config.Config{DataPath: t.TempDir()})
+ if err != nil {
+ t.Fatalf("new monitor: %v", err)
+ }
+ t.Cleanup(func() { monitor.Stop() })
+ return monitor
+}
+
+// Wiring a monitor attaches its store and every supplemental provider. Each
+// separate wiring step published the whole estate, so a router with two
+// platform providers ran three full store passes (and the first two read an
+// incomplete provider set) before the listener opened.
+func TestConfigureMonitorDependenciesPublishesTheEstateOnce(t *testing.T) {
+ trueNAS := &countingSupplementalRecords{}
+ vmware := &countingSupplementalRecords{}
+ router := &Router{
+ monitorResourceAdapter: unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil)),
+ monitorResourceAdapters: make(map[string]*unifiedresources.MonitorAdapter),
+ monitorSupplementalRecords: map[unifiedresources.DataSource]monitoring.MonitorSupplementalRecordsProvider{
+ unifiedresources.SourceTrueNAS: trueNAS,
+ unifiedresources.SourceVMware: vmware,
+ },
+ }
+
+ monitor := newWiringTestMonitor(t)
+ router.configureMonitorDependencies(monitor)
+
+ if got := trueNAS.reads.Load(); got != 1 {
+ t.Fatalf("TrueNAS provider read %d times while wiring one monitor, want 1 store pass", got)
+ }
+ if got := vmware.reads.Load(); got != 1 {
+ t.Fatalf("VMware provider read %d times while wiring one monitor, want 1 store pass", got)
+ }
+
+ // NewRouter, SetMonitor, SetMultiTenantMonitor and every provider change
+ // wire the same monitor again; none of it has anything new to publish.
+ router.configureMonitorDependencies(monitor)
+ router.configureMonitorDependencies(monitor)
+ if trueNAS.reads.Load() != 1 || vmware.reads.Load() != 1 {
+ t.Fatalf("re-wiring the same monitor read the providers %d and %d times in all, want one store pass", trueNAS.reads.Load(), vmware.reads.Load())
+ }
+}
+
+// passCountingStore counts the registry rebuilds a monitor publishes into it.
+type passCountingStore struct {
+ *unifiedresources.MonitorAdapter
+ passes atomic.Int32
+}
+
+func (s *passCountingStore) PopulateSnapshotAndSupplemental(snapshot models.StateSnapshot, recordsBySource map[unifiedresources.DataSource][]unifiedresources.IngestRecord) {
+ s.passes.Add(1)
+ s.MonitorAdapter.PopulateSnapshotAndSupplemental(snapshot, recordsBySource)
+}
+
+// A mock-mode switch re-registers both platform providers; the running monitor
+// must publish once for the pair, not once per source.
+func TestSyncPlatformSupplementalProvidersPublishesOnceForBothSources(t *testing.T) {
+ previousTrueNAS := truenas.IsFeatureEnabled()
+ previousVMware := vmware.IsFeatureEnabled()
+ t.Cleanup(func() {
+ truenas.SetFeatureEnabled(previousTrueNAS)
+ vmware.SetFeatureEnabled(previousVMware)
+ })
+
+ monitor := newWiringTestMonitor(t)
+ store := &passCountingStore{MonitorAdapter: unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil))}
+ monitor.SetResourceStore(store)
+ router := &Router{monitor: monitor}
+
+ before := store.passes.Load()
+ router.syncPlatformSupplementalProviders(true)
+ if got := store.passes.Load() - before; got != 1 {
+ t.Fatalf("switching mock mode on published %d times, want 1 for the TrueNAS and VMware pair", got)
+ }
+
+ before = store.passes.Load()
+ router.syncPlatformSupplementalProviders(false)
+ if got := store.passes.Load() - before; got != 1 {
+ t.Fatalf("switching mock mode off published %d times, want 1 for the TrueNAS and VMware pair", got)
+ }
+}
+
+// This uses a real alert on the first publication, not a later provider read:
+// saved policy must be installed before any alert reaches the lifecycle hook.
+func TestConfigureMonitorDependenciesAppliesPolicyBeforeFirstAlert(t *testing.T) {
+ monitor := newWiringTestMonitor(t)
+ manager := monitor.GetAlertManager()
+ cfg := manager.GetConfig()
+ cfg.Enabled = false
+ manager.UpdateConfig(cfg)
+ // Install a cached source-native provider before a store is attached. Its
+ // first publication evaluates a real TrueNAS memory threshold; a normal
+ // host-agent row is checked on its separate legacy alert path instead.
+ provider := &alertWiringProvider{}
+ monitor.SetSupplementalRecordsProvider(unifiedresources.SourceTrueNAS, provider)
+ cfg = manager.GetConfig()
+ cfg.Enabled = true
+ cfg.ActivationState = alerts.ActivationActive
+ cfg.TimeThresholds = map[string]int{}
+ cfg.MetricTimeThresholds = nil
+ cfg.TrueNASDefaults.Memory = &alerts.HysteresisThreshold{Trigger: 1, Clear: 0.5}
+ manager.UpdateConfig(cfg)
+ persistence := config.NewConfigPersistence(t.TempDir())
+ settings := config.DefaultSystemSettings()
+ settings.PublicURL = "https://pulse.example.test"
+ settings.WebhookAllowedPrivateCIDRs = "192.168.50.0/24"
+ if err := persistence.SaveSystemSettings(*settings); err != nil {
+ t.Fatal(err)
+ }
+ notifications := monitor.GetNotificationManager()
+ if notifications.ValidateWebhookURL("http://192.168.50.2/hook") == nil {
+ t.Fatal("fresh manager already allows the saved private network")
+ }
+ observations := 0
+ unsubscribe := manager.SubscribeLifecycleCallback(func(alerts.LifecycleEvent) {
+ observations++
+ if got := notifications.GetPublicURL(); got != settings.PublicURL {
+ t.Errorf("first-publication alert used PublicURL %q, want %q", got, settings.PublicURL)
+ }
+ if err := notifications.ValidateWebhookURL("http://192.168.50.2/hook"); err != nil {
+ t.Errorf("saved allowlist absent at first-publication alert: %v", err)
+ }
+ if notifications.ValidateWebhookURL("http://192.168.51.2/hook") == nil {
+ t.Error("unlisted private network was admitted")
+ }
+ })
+ defer unsubscribe()
+ router := &Router{persistence: persistence, monitorResourceAdapter: unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil))}
+ router.configureMonitorDependencies(monitor)
+ if observations == 0 {
+ resources := wiringResources(monitor)
+ if len(resources) != 1 {
+ t.Fatalf("no alert; resources = %v", resources)
+ }
+ input, _ := alerts.UnifiedResourceInputFromResource(resources[0])
+ t.Fatalf("first publication emitted no lifecycle event; input=%+v policy=%+v", input, manager.GetConfig().AgentDefaults)
+ }
+}
+
+type tenantWiringProvider struct{ prefix string }
+
+func (p *tenantWiringProvider) SupplementalRecords(_ *monitoring.Monitor, org string) []unifiedresources.IngestRecord {
+ name := p.prefix + "-" + org
+ return []unifiedresources.IngestRecord{{SourceID: name, Resource: unifiedresources.Resource{Type: unifiedresources.ResourceTypeAgent, Name: name, Status: unifiedresources.StatusOnline, LastSeen: time.Now().UTC()}, Identity: unifiedresources.ResourceIdentity{Hostnames: []string{name}}}}
+}
+
+func TestRouterResourceWiringWithdrawsProvidersFromEveryExistingAndFutureTenant(t *testing.T) {
+ persistence := config.NewMultiTenantPersistence(t.TempDir())
+ for _, org := range []string{"tenant-a", "tenant-b", "tenant-c"} {
+ if err := persistence.SaveOrganization(&models.Organization{ID: org}); err != nil {
+ t.Fatal(err)
+ }
+ }
+ mt := monitoring.NewMultiTenantMonitor(&config.Config{DataPath: t.TempDir()}, persistence, nil)
+ defer mt.Stop()
+ router := &Router{mtMonitor: mt}
+ provider := &tenantWiringProvider{prefix: "source"}
+ router.setMonitorSupplementalRecordsProvider(unifiedresources.SourceTrueNAS, provider)
+ for _, org := range []string{"tenant-a", "tenant-b"} {
+ m, err := mt.GetMonitor(org)
+ if err != nil {
+ t.Fatal(err)
+ }
+ resources := wiringResources(m)
+ if len(resources) != 1 || resources[0].Name != "source-"+org {
+ t.Fatalf("%s registry = %v", org, resources)
+ }
+ }
+ router.setMonitorSupplementalRecordsProvider(unifiedresources.SourceTrueNAS, nil)
+ for _, org := range []string{"tenant-a", "tenant-b", "tenant-c"} {
+ m, err := mt.GetMonitor(org)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if resources := wiringResources(m); len(resources) != 0 {
+ t.Fatalf("%s retained withdrawn provider: %v", org, resources)
+ }
+ }
+}
+
+func TestRouterResourceWiringSerializesInitializationAndWithdrawal(t *testing.T) {
+ monitor := newWiringTestMonitor(t)
+ entered, release, initialized, withdrawn := make(chan struct{}), make(chan struct{}), make(chan struct{}), make(chan struct{})
+ var once sync.Once
+ unblock := func() { once.Do(func() { close(release) }) }
+ defer unblock()
+ provider := &blockingRouterWiringProvider{entered: entered, release: release}
+ router := &Router{monitor: monitor, monitorResourceAdapter: unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil)), monitorSupplementalRecords: map[unifiedresources.DataSource]monitoring.MonitorSupplementalRecordsProvider{unifiedresources.SourceTrueNAS: provider}}
+ go func() { defer close(initialized); router.configureMonitorDependencies(monitor) }()
+ select {
+ case <-entered:
+ case <-time.After(5 * time.Second):
+ t.Fatal("initializer did not collect")
+ }
+ go func() {
+ defer close(withdrawn)
+ router.setMonitorSupplementalRecordsProvider(unifiedresources.SourceTrueNAS, nil)
+ }()
+ // Check the exact exclusion boundary while the initializer is blocked.
+ if router.monitorSupplementalMu.TryLock() {
+ router.monitorSupplementalMu.Unlock()
+ t.Fatal("initializer released its provider registry before applying it")
+ }
+ unblock()
+ select {
+ case <-initialized:
+ case <-time.After(5 * time.Second):
+ t.Fatal("initializer did not finish")
+ }
+ select {
+ case <-withdrawn:
+ case <-time.After(5 * time.Second):
+ t.Fatal("withdrawal did not finish")
+ }
+ if resources := wiringResources(monitor); len(resources) != 0 {
+ t.Fatalf("withdrawal left rows: %v", resources)
+ }
+}
+
+type blockingRouterWiringProvider struct{ entered, release chan struct{} }
+
+func (p *blockingRouterWiringProvider) SupplementalRecords(*monitoring.Monitor, string) []unifiedresources.IngestRecord {
+ close(p.entered)
+ <-p.release
+ return (&tenantWiringProvider{prefix: "old"}).SupplementalRecords(nil, "default")
+}
+
+func wiringResources(m *monitoring.Monitor) []unifiedresources.Resource {
+ return m.GetUnifiedReadState().(interface {
+ GetAll() []unifiedresources.Resource
+ }).GetAll()
+}
+
+func TestConfigureMonitorDependenciesRevokesEmptySavedAllowlist(t *testing.T) {
+ monitor := newWiringTestMonitor(t)
+ persistence := config.NewConfigPersistence(t.TempDir())
+ settings := config.DefaultSystemSettings()
+ settings.PublicURL = "https://pulse.example.test"
+ settings.WebhookAllowedPrivateCIDRs = "192.168.50.0/24"
+ if err := persistence.SaveSystemSettings(*settings); err != nil {
+ t.Fatal(err)
+ }
+ router := &Router{persistence: persistence, monitorResourceAdapter: unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil))}
+ router.configureMonitorDependencies(monitor)
+ nm := monitor.GetNotificationManager()
+ if nm.GetPublicURL() != settings.PublicURL || nm.ValidateWebhookURL("http://192.168.50.2/hook") != nil {
+ t.Fatal("saved policy did not apply")
+ }
+ settings.WebhookAllowedPrivateCIDRs = ""
+ if err := persistence.SaveSystemSettings(*settings); err != nil {
+ t.Fatal(err)
+ }
+ router.configureMonitorDependencies(monitor)
+ if nm.GetPublicURL() != settings.PublicURL || nm.ValidateWebhookURL("http://192.168.50.2/hook") == nil {
+ t.Fatal("reinitialization retained revoked notification policy")
+ }
+}
+
+// A cached, synthetic source-native observation. No provider request is made.
+type alertWiringProvider struct{}
+
+func (*alertWiringProvider) SupplementalRecords(*monitoring.Monitor, string) []unifiedresources.IngestRecord {
+ return []unifiedresources.IngestRecord{{SourceID: "policy-proof-host", Resource: unifiedresources.Resource{
+ Type: unifiedresources.ResourceTypeAgent, Name: "policy-proof-host", Status: unifiedresources.StatusOnline, LastSeen: time.Now().UTC(),
+ Sources: []unifiedresources.DataSource{unifiedresources.SourceTrueNAS},
+ Metrics: &unifiedresources.ResourceMetrics{Memory: &unifiedresources.MetricValue{Value: 99, Percent: 99, Unit: "percent"}},
+ }, Identity: unifiedresources.ResourceIdentity{Hostnames: []string{"policy-proof-host"}}}}
+}
diff --git a/internal/monitoring/mock_mode_fence.go b/internal/monitoring/mock_mode_fence.go
index c552480bbe..9699a275d5 100644
--- a/internal/monitoring/mock_mode_fence.go
+++ b/internal/monitoring/mock_mode_fence.go
@@ -5,6 +5,11 @@ import (
"sync/atomic"
)
+// Store/provider rewiring also advances this epoch: its old inventory must
+// not reach alerts after the new dependency binding is published. Unlike a
+// mode switch, rewiring does not clear active alerts. Both callers join only
+// admitted alert-manager calls, never a slow provider read or registry write.
+//
// mockModeFence keeps alert evaluations of mode-dependent data inside the
// mock-mode epoch they read it in. GetState, the fixture graph, recovery
// rollups and the connection ledger all switch source when mock mode flips,
diff --git a/internal/monitoring/monitor.go b/internal/monitoring/monitor.go
index 6e76f913f9..9977be5ec5 100644
--- a/internal/monitoring/monitor.go
+++ b/internal/monitoring/monitor.go
@@ -1426,6 +1426,9 @@ type Monitor struct {
nodePendingUpdatesCache map[string]pendingUpdatesCache // Cache pending updates per node (checked every 30 min)
pveMembershipMisses map[string]int // Consecutive authoritative membership absences by instance/node
resourceStore ResourceStoreInterface // Optional unified resource store for polling optimization
+ resourceWiringMu sync.Mutex // Serializes dependency changes and their attachment effects, not provider/alert callbacks
+ resourceWiringGeneration atomic.Uint64 // Store/provider binding captured by a refresh
+ resourcePublishMu sync.Mutex // Serializes full publications after provider collection
supplementalProviders map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider
recoveryManager *recoverymanager.Manager // Optional recovery store manager for backup rollups
recoveryIngestMu sync.Mutex
@@ -5028,73 +5031,147 @@ func (m *Monitor) DisableTemperatureMonitoring() {
log.Info().Msg("temperature monitoring disabled")
}
-// SetResourceStore sets the resource store for polling optimization.
-// When set, the monitor will check if it should reduce polling frequency
-// for nodes that have host agents providing data.
+// SetResourceStore sets the resource store and immediately backfills it, even
+// when the same store is supplied again. Use SetResourceStoreAndProviders for
+// repeatable router/tenant initialization without redundant full refreshes.
func (m *Monitor) SetResourceStore(store ResourceStoreInterface) {
+ m.wireResourceStoreAndProviders(store, nil, true)
+}
+
+// SetResourceStoreAndProviders attaches the store and a batch of provider
+// updates in one critical section, then publishes once if anything changed.
+// Omitted sources are unchanged; an explicit nil withdraws a source. Like a
+// mock-mode switch, changing dependencies must not run inside an admitted
+// alert callback: it joins old alert calls before the new binding can publish.
+func (m *Monitor) SetResourceStoreAndProviders(store ResourceStoreInterface, providers map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider) {
+ m.wireResourceStoreAndProviders(store, providers, false)
+}
+
+// sameWiredValue treats uncomparable dependency values as changed, never as a
+// reason to panic or silently skip a refresh.
+func sameWiredValue(a, b any) (same bool) {
+ defer func() {
+ if recover() != nil {
+ same = false
+ }
+ }()
+ return a == b
+}
+
+func (m *Monitor) wireResourceStoreAndProviders(store ResourceStoreInterface, providers map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider, force bool) {
+ if m == nil {
+ return
+ }
+ m.resourceWiringMu.Lock()
if thresholdStore, ok := store.(StaleThresholdResourceStore); ok {
thresholdStore.SetStaleThresholds(m.resourceStaleThresholds())
}
m.mu.Lock()
+ _, providersChanged := m.registerSupplementalProvidersLocked(providers)
+ storeChanged := !sameWiredValue(m.resourceStore, store)
m.resourceStore = store
incidentStore := m.incidentStore
m.installNodeAgentSplitDeciderLocked()
+ changed := storeChanged || providersChanged
+ if changed {
+ m.resourceWiringGeneration.Add(1)
+ }
m.mu.Unlock()
- m.installOperatorIntentResolver(store)
- log.Info().Msg("resource store set for polling optimization")
- timelineAttached := false
- if incidentStore != nil {
- if timelineStore, ok := store.(memory.IncidentTimelineStore); ok {
+ if storeChanged || force {
+ m.installOperatorIntentResolver(store)
+ if incidentStore != nil {
+ timelineStore, _ := store.(memory.IncidentTimelineStore)
incidentStore.SetResourceTimelineStore(timelineStore)
- timelineAttached = true
- } else {
- incidentStore.SetResourceTimelineStore(nil)
+ if timelineStore != nil {
+ // Router/health serving must not wait for backlog projection.
+ m.scheduleAlertProjectionCatchUp()
+ }
}
}
- if timelineAttached {
- // The canonical boundary: with the durable resource store attached,
- // restart repair can reach resource history as well as the incident
- // fallback cache, and the replay watermark may advance. Run it in the
- // background — router construction and health serving must not wait
- // behind an un-projected backlog.
- m.scheduleAlertProjectionCatchUp()
+ m.resourceWiringMu.Unlock()
+
+ if changed {
+ // The epoch guards dependency-derived alerts as well as mode-derived
+ // ones. No active alert is cleared here; the new inventory evaluates
+ // them normally. No monitor/state lock is held while joining old calls.
+ m.mockModeFence.advance()
+ }
+
+ if changed || force {
+ m.backfillResourceStore(false)
}
+}
- // Immediately backfill the store from current state so ReadState
- // consumers have data as soon as the store is wired.
- // Guard against minimally initialized monitors (e.g., test fixtures
- // with bare &Monitor{}) where m.state may be nil.
- if store != nil && m.state != nil {
+func (m *Monitor) backfillResourceStore(allowEmptyState bool) {
+ // Bare monitor fixtures have no state. Read the pointer under the same
+ // lock used by a mock switch replacing it.
+ m.mu.RLock()
+ ready := m.resourceStore != nil && (m.state != nil || allowEmptyState)
+ m.mu.RUnlock()
+ if ready {
m.updateResourceStore(m.currentStateWithScope())
}
}
-// SetSupplementalRecordsProvider configures source-native resource providers
-// that ingest alongside the legacy state snapshot path.
+// SetSupplementalRecordsProvider preserves the single-source explicit-refresh
+// behavior, including when the supplied provider is unchanged.
func (m *Monitor) SetSupplementalRecordsProvider(source unifiedresources.DataSource, provider MonitorSupplementalRecordsProvider) {
- if m == nil {
- return
- }
+ m.SetSupplementalRecordsProviders(map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider{source: provider})
+}
- normalized := unifiedresources.DataSource(strings.ToLower(strings.TrimSpace(string(source))))
- if normalized == "" {
+// SetSupplementalRecordsProviders applies a batch of source updates and
+// refreshes once. Empty source names are ignored; nil withdraws a provider.
+// Changed bindings have the same alert-callback restriction as the combined setter.
+func (m *Monitor) SetSupplementalRecordsProviders(providers map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider) {
+ if m == nil {
return
}
-
+ m.resourceWiringMu.Lock()
m.mu.Lock()
- if m.supplementalProviders == nil {
- m.supplementalProviders = make(map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider)
- }
- if provider == nil {
- delete(m.supplementalProviders, normalized)
- } else {
- m.supplementalProviders[normalized] = provider
+ usable, changed := m.registerSupplementalProvidersLocked(providers)
+ if changed {
+ m.resourceWiringGeneration.Add(1)
}
m.mu.Unlock()
+ m.resourceWiringMu.Unlock()
+ if changed {
+ m.mockModeFence.advance()
+ }
+ if usable {
+ // Source-native providers can publish into a store even before a
+ // legacy state exists (including minimally initialized monitors).
+ m.backfillResourceStore(true)
+ }
+}
- m.updateResourceStore(m.currentStateWithScope())
+// The caller holds m.mu. Compare and update every source in the same section
+// as the store binding, so a refresh can never capture half a wiring batch.
+func (m *Monitor) registerSupplementalProvidersLocked(providers map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider) (usable, changed bool) {
+ for source, provider := range providers {
+ normalized := unifiedresources.DataSource(strings.ToLower(strings.TrimSpace(string(source))))
+ if normalized == "" {
+ continue
+ }
+ usable = true
+ previous, registered := m.supplementalProviders[normalized]
+ if provider == nil {
+ if registered {
+ delete(m.supplementalProviders, normalized)
+ changed = true
+ }
+ continue
+ }
+ if !registered || !sameWiredValue(previous, provider) {
+ changed = true
+ }
+ if m.supplementalProviders == nil {
+ m.supplementalProviders = make(map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider)
+ }
+ m.supplementalProviders[normalized] = provider
+ }
+ return usable, changed
}
// SetLicenseChecker wires the commercial feature gate used by monitoring-owned
@@ -5677,6 +5754,7 @@ func (m *Monitor) currentStateWithScope() (models.StateSnapshot, mockModeScope)
func (m *Monitor) updateResourceStore(state models.StateSnapshot, scope mockModeScope) {
m.mu.RLock()
store := m.resourceStore
+ generation := m.resourceWiringGeneration.Load()
m.mu.RUnlock()
if store == nil {
@@ -5709,12 +5787,20 @@ func (m *Monitor) updateResourceStore(state models.StateSnapshot, scope mockMode
Msg("[Resources] Suppressing legacy snapshot slices for provider-owned sources")
}
+ // Provider callbacks can read or re-enter the monitor. Collect without any
+ // publication/wiring lock, then refuse a binding that changed meanwhile.
+ if !scope.current() {
+ return
+ }
+ recordsBySource := m.collectSupplementalRecordsBySource()
+ m.resourcePublishMu.Lock()
+ if m.resourceWiringGeneration.Load() != generation {
+ m.resourcePublishMu.Unlock()
+ return
+ }
mark, published := scope.publish(func() bool {
- recordsBySource := m.collectSupplementalRecordsBySource()
- supplementalChanges := m.collectSupplementalChanges()
if atomicStore, ok := store.(AtomicSnapshotResourceStore); ok {
atomicStore.PopulateSnapshotAndSupplemental(snapshotForStore, recordsBySource)
- recordSupplementalResourceChanges(store, supplementalChanges)
for source, records := range recordsBySource {
if len(records) == 0 {
continue
@@ -5743,14 +5829,20 @@ func (m *Monitor) updateResourceStore(state models.StateSnapshot, scope mockMode
}
}
- recordSupplementalResourceChanges(store, supplementalChanges)
return true
})
- if !published {
+ m.resourcePublishMu.Unlock()
+ if !published || m.resourceWiringGeneration.Load() != generation {
// The snapshot belongs to the mode the monitor just left.
return
}
- m.syncPublishedResourceStore(store, scope, mark)
+ // Drain consume-once changes only for a pass that actually published. No
+ // provider callback runs under resourcePublishMu, and changes already
+ // drained remain history even if a subsequent rewire replaces the binding.
+ recordSupplementalResourceChanges(store, m.collectSupplementalChanges())
+ if m.resourceWiringGeneration.Load() == generation {
+ m.syncPublishedResourceStore(store, scope, mark)
+ }
}
// resourceSnapshotStore serves one GetAll clone to every consumer of a single
diff --git a/internal/monitoring/monitor_mock_alerts_test.go b/internal/monitoring/monitor_mock_alerts_test.go
index 4b54aa19e5..6029a8726e 100644
--- a/internal/monitoring/monitor_mock_alerts_test.go
+++ b/internal/monitoring/monitor_mock_alerts_test.go
@@ -694,7 +694,7 @@ func alertTypes(active []alerts.Alert) map[string]int {
return types
}
-func TestLiveRefreshIgnoresARegistryRepublishedFromMockMode(t *testing.T) {
+func TestLiveRefreshRejectsCollectedRecordsFromMockMode(t *testing.T) {
setMockSamplerTestEnv(t, time.Hour, 5*time.Minute)
pinDefaultMockEstate(t)
manager := newMockHostAlertTestManager(t)
@@ -758,7 +758,8 @@ func TestLiveRefreshIgnoresARegistryRepublishedFromMockMode(t *testing.T) {
}
}
- // The mock-mode refresh now publishes the fixture estate over it.
+ // The old collector finishes, but must be refused before publication.
+ // Collection no longer owns a publication admission spanning the switch.
close(provider.release)
releasedProvider = true
waitForSignal(t, staleDone, 30*time.Second, "mock-mode refresh did not finish after it was released")
@@ -766,20 +767,17 @@ func TestLiveRefreshIgnoresARegistryRepublishedFromMockMode(t *testing.T) {
releasedList = true
waitForSignal(t, liveDone, 30*time.Second, "live refresh did not finish after it was released")
- if active := manager.GetActiveAlerts(); len(active) != 0 {
- t.Fatalf("a live refresh evaluated a registry the mock-mode refresh republished, leaving %d fixture alerts: %s",
- len(active), describeAlerts(active))
- }
- if monitor.mockModeFence.registryCurrent() {
- t.Fatal("the registry the mock-mode refresh republished counted as current")
+ for _, resource := range store.MonitorAdapter.GetAll() {
+ if slices.Contains(resource.Sources, unifiedresources.SourceVMware) {
+ t.Fatalf("the ended collector republished vSphere fixture resource %s", resource.ID)
+ }
}
-
- // The next live refresh is undisturbed: it replaces the registry and
- // evaluates the live records.
- monitor.updateResourceStore(monitor.currentStateWithScope())
if !monitor.mockModeFence.registryCurrent() {
- t.Fatal("an undisturbed live rebuild did not make the registry current")
+ t.Fatal("the refused collector disturbed the current live publication")
}
+
+ // The admitted live pass still evaluates its real source-native records;
+ // refusal of old collections must not become blanket alert suppression.
types := alertTypes(manager.GetActiveAlerts())
if types["zfs-pool-state"] == 0 {
t.Fatalf("the undisturbed live refresh raised no alert from the live TrueNAS records: %v", types)
diff --git a/internal/monitoring/poll_providers.go b/internal/monitoring/poll_providers.go
index 212e8a2b12..ed7d2fd1c3 100644
--- a/internal/monitoring/poll_providers.go
+++ b/internal/monitoring/poll_providers.go
@@ -6,6 +6,7 @@ import (
"strings"
"time"
+ "github.com/rcourtman/pulse-go-rewrite/internal/mock"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
"github.com/rcourtman/pulse-go-rewrite/pkg/pbs"
"github.com/rcourtman/pulse-go-rewrite/pkg/pmg"
@@ -913,6 +914,14 @@ func (m *Monitor) supplementalProviderSnapshot() map[unifiedresources.DataSource
}
m.mu.RUnlock()
+ // The flag flips before the router's post-switch hook runs. Use one mode
+ // for the pair, and keep these optional callbacks outside the monitor lock.
+ mode := mock.IsMockEnabled()
+ for source, provider := range snapshot {
+ if bound, ok := provider.(interface{ SupplementalRecordsMockMode() bool }); ok && bound.SupplementalRecordsMockMode() != mode {
+ delete(snapshot, source)
+ }
+ }
if len(snapshot) == 0 {
return nil
}
diff --git a/internal/monitoring/resource_wiring_test.go b/internal/monitoring/resource_wiring_test.go
new file mode 100644
index 0000000000..eff983e2e6
--- /dev/null
+++ b/internal/monitoring/resource_wiring_test.go
@@ -0,0 +1,458 @@
+package monitoring
+
+import (
+ "github.com/rcourtman/pulse-go-rewrite/internal/alerts"
+ "github.com/rcourtman/pulse-go-rewrite/internal/mock"
+ "github.com/rcourtman/pulse-go-rewrite/internal/models"
+ "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+)
+
+func storeWiringProvider(sourceID string) *testMonitorSupplementalProvider {
+ return &testMonitorSupplementalProvider{
+ recordsByOrg: map[string][]unifiedresources.IngestRecord{
+ "default": {{
+ SourceID: sourceID,
+ Resource: unifiedresources.Resource{
+ Type: unifiedresources.ResourceTypeAgent,
+ Name: sourceID,
+ Status: unifiedresources.StatusOnline,
+ LastSeen: time.Now().UTC(),
+ },
+ Identity: unifiedresources.ResourceIdentity{Hostnames: []string{sourceID}},
+ }},
+ },
+ }
+}
+
+// Every wiring call that publishes populates the whole registry and runs the
+// alert pass over every resource, so startup, which wires a store and one
+// provider per supplemental source, must publish once, not once per source.
+func TestSetResourceStoreAndProvidersPublishesOnce(t *testing.T) {
+ store := &testSupplementalResourceStore{}
+ monitor := &Monitor{state: models.NewState()}
+
+ monitor.SetResourceStoreAndProviders(store, map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider{
+ unifiedresources.SourceTrueNAS: storeWiringProvider("tn-host-1"),
+ unifiedresources.SourceVMware: storeWiringProvider("esxi-host-1"),
+ })
+
+ if store.snapshotCalls != 1 {
+ t.Fatalf("wiring a store and two providers populated the registry %d times, want 1", store.snapshotCalls)
+ }
+ for source, sourceID := range map[unifiedresources.DataSource]string{
+ unifiedresources.SourceTrueNAS: "tn-host-1",
+ unifiedresources.SourceVMware: "esxi-host-1",
+ } {
+ records := store.recordsBySource[source]
+ if len(records) != 1 || records[0].SourceID != sourceID {
+ t.Fatalf("the one pass ingested %v for %s, want the %s record", records, source, sourceID)
+ }
+ }
+}
+
+// A router re-applies its wiring to a monitor it already wired (a provider
+// change re-runs the tenant initializer on every existing monitor, and so do
+// SetMonitor and SetMultiTenantMonitor). The same store and providers must not
+// repopulate the registry again; a different provider or store must.
+func TestSetResourceStoreAndProvidersPublishesOnlyWhenWiringChanged(t *testing.T) {
+ store := &testSupplementalResourceStore{}
+ monitor := &Monitor{state: models.NewState()}
+ trueNAS := storeWiringProvider("tn-host-1")
+ vmware := storeWiringProvider("esxi-host-1")
+ providers := map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider{
+ unifiedresources.SourceTrueNAS: trueNAS,
+ unifiedresources.SourceVMware: vmware,
+ }
+
+ monitor.SetResourceStoreAndProviders(store, providers)
+ monitor.SetResourceStoreAndProviders(store, providers)
+ monitor.SetResourceStoreAndProviders(store, map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider{unifiedresources.SourceVMware: vmware})
+ if store.snapshotCalls != 1 {
+ t.Fatalf("re-wiring the same store and providers populated the registry %d times in all, want 1", store.snapshotCalls)
+ }
+
+ monitor.SetResourceStoreAndProviders(store, map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider{
+ unifiedresources.SourceVMware: storeWiringProvider("esxi-host-2"),
+ })
+ if store.snapshotCalls != 2 {
+ t.Fatalf("wiring a different provider populated the registry %d times in all, want 2", store.snapshotCalls)
+ }
+
+ monitor.SetResourceStoreAndProviders(&testSupplementalResourceStore{}, nil)
+ if store.snapshotCalls != 2 {
+ t.Fatalf("the replaced store was populated again: %d calls", store.snapshotCalls)
+ }
+ replacement, ok := monitor.resourceStore.(*testSupplementalResourceStore)
+ if !ok || replacement == store {
+ t.Fatalf("the monitor kept the store it was wired with: %#v", monitor.resourceStore)
+ }
+ if replacement.snapshotCalls != 1 {
+ t.Fatalf("wiring a different store populated it %d times, want 1", replacement.snapshotCalls)
+ }
+}
+
+// A provider type that cannot be compared must read as a change, not panic.
+type uncomparableWiringProvider struct {
+ records []unifiedresources.IngestRecord
+}
+
+func (uncomparableWiringProvider) SupplementalRecords(*Monitor, string) []unifiedresources.IngestRecord {
+ return nil
+}
+
+func TestSetResourceStoreAndProvidersTreatsUncomparableProvidersAsChanged(t *testing.T) {
+ store := &testSupplementalResourceStore{}
+ monitor := &Monitor{state: models.NewState()}
+ providers := map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider{
+ unifiedresources.SourceTrueNAS: uncomparableWiringProvider{},
+ }
+
+ monitor.SetResourceStoreAndProviders(store, providers)
+ monitor.SetResourceStoreAndProviders(store, providers)
+
+ if store.snapshotCalls != 2 {
+ t.Fatalf("an uncomparable provider populated the registry %d times for two wirings, want 2", store.snapshotCalls)
+ }
+}
+
+func TestSetSupplementalRecordsProvidersPublishesOnce(t *testing.T) {
+ store := &testSupplementalResourceStore{}
+ monitor := &Monitor{state: models.NewState(), resourceStore: store}
+
+ monitor.SetSupplementalRecordsProviders(map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider{
+ unifiedresources.SourceTrueNAS: storeWiringProvider("tn-host-1"),
+ unifiedresources.SourceVMware: storeWiringProvider("esxi-host-1"),
+ })
+
+ if store.snapshotCalls != 1 {
+ t.Fatalf("registering two providers populated the registry %d times, want 1", store.snapshotCalls)
+ }
+ if len(store.recordsBySource[unifiedresources.SourceTrueNAS]) != 1 || len(store.recordsBySource[unifiedresources.SourceVMware]) != 1 {
+ t.Fatalf("the pass did not ingest both providers: %v", store.recordsBySource)
+ }
+}
+
+func TestSetSupplementalRecordsProvidersNormalizesRemovesAndSkipsEmpty(t *testing.T) {
+ store := &testSupplementalResourceStore{}
+ monitor := &Monitor{state: models.NewState(), resourceStore: store}
+
+ monitor.SetSupplementalRecordsProviders(map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider{
+ " TrueNAS ": storeWiringProvider("tn-host-1"),
+ " ": storeWiringProvider("nameless"),
+ })
+ if store.snapshotCalls != 1 {
+ t.Fatalf("a usable source populated the registry %d times, want 1", store.snapshotCalls)
+ }
+ if len(store.recordsBySource[unifiedresources.SourceTrueNAS]) != 1 {
+ t.Fatalf("a provider registered as %q was not found under %q: %v", " TrueNAS ", unifiedresources.SourceTrueNAS, store.recordsBySource)
+ }
+ if len(store.recordsBySource) != 1 {
+ t.Fatalf("a blank source was registered: %v", store.recordsBySource)
+ }
+
+ store.snapshotCalls = 0
+ monitor.SetSupplementalRecordsProviders(map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider{" ": storeWiringProvider("nameless")})
+ monitor.SetSupplementalRecordsProviders(nil)
+ if store.snapshotCalls != 0 {
+ t.Fatalf("a change naming no usable source populated the registry %d times, want 0", store.snapshotCalls)
+ }
+
+ store.recordsBySource = nil
+ monitor.SetSupplementalRecordsProviders(map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider{unifiedresources.SourceTrueNAS: nil})
+ if store.snapshotCalls != 1 {
+ t.Fatalf("removing a provider populated the registry %d times, want 1", store.snapshotCalls)
+ }
+ if len(store.recordsBySource) != 0 {
+ t.Fatalf("a removed provider still supplied records: %v", store.recordsBySource)
+ }
+}
+
+// The single-source setters keep publishing, so a caller that wires one piece
+// at a time still ends with a populated registry.
+func TestSingleWiringSettersStillPublish(t *testing.T) {
+ store := &testSupplementalResourceStore{}
+ monitor := &Monitor{state: models.NewState()}
+
+ monitor.SetResourceStore(store)
+ if store.snapshotCalls != 1 {
+ t.Fatalf("SetResourceStore populated the registry %d times, want 1", store.snapshotCalls)
+ }
+
+ provider := storeWiringProvider("tn-host-1")
+ monitor.SetSupplementalRecordsProvider(unifiedresources.SourceTrueNAS, provider)
+ if store.snapshotCalls != 2 {
+ t.Fatalf("SetSupplementalRecordsProvider populated the registry %d times in all, want 2", store.snapshotCalls)
+ }
+ if len(store.recordsBySource[unifiedresources.SourceTrueNAS]) != 1 {
+ t.Fatalf("the provider's records were not ingested: %v", store.recordsBySource)
+ }
+
+ // Unlike the batched wiring step, the single setters are a refresh the
+ // caller asked for: the same arguments publish again.
+ monitor.SetResourceStore(store)
+ monitor.SetSupplementalRecordsProvider(unifiedresources.SourceTrueNAS, provider)
+ if store.snapshotCalls != 4 {
+ t.Fatalf("repeating the single setters populated the registry %d times in all, want 4", store.snapshotCalls)
+ }
+}
+
+// This store uses the real in-memory adapter and records every publication,
+// rather than only checking the final rows (which could conceal a stale pass).
+type wiringPublicationStore struct {
+ *unifiedresources.MonitorAdapter
+ mu sync.Mutex
+ batches [][]string
+ before func()
+}
+
+func (s *wiringPublicationStore) PopulateSnapshotAndSupplemental(snapshot models.StateSnapshot, records map[unifiedresources.DataSource][]unifiedresources.IngestRecord) {
+ if s.before != nil {
+ s.before()
+ }
+ names := []string{}
+ for _, batch := range records {
+ for _, record := range batch {
+ names = append(names, record.SourceID)
+ }
+ }
+ s.mu.Lock()
+ s.batches = append(s.batches, names)
+ s.mu.Unlock()
+ s.MonitorAdapter.PopulateSnapshotAndSupplemental(snapshot, records)
+}
+
+func newWiringPublicationStore() *wiringPublicationStore {
+ return &wiringPublicationStore{MonitorAdapter: unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil))}
+}
+
+type wiringRecordsFunc func(*Monitor, string) []unifiedresources.IngestRecord
+
+func (f wiringRecordsFunc) SupplementalRecords(m *Monitor, org string) []unifiedresources.IngestRecord {
+ return f(m, org)
+}
+
+func wiringRecords(name string) []unifiedresources.IngestRecord {
+ return storeWiringProvider(name).recordsByOrg["default"]
+}
+
+// Registration owns a new refresh. A delayed collector is refused before it
+// can publish old rows, not repaired by publishing them and repeating a pass.
+func TestResourceWiringRefusesCollectedReplacedOrWithdrawnProviders(t *testing.T) {
+ for _, withdraw := range []bool{false, true} {
+ name := "replace"
+ if withdraw {
+ name = "withdraw"
+ }
+ t.Run(name, func(t *testing.T) {
+ store := newWiringPublicationStore()
+ m := &Monitor{state: models.NewState(), resourceStore: store}
+ entered, release, done := make(chan struct{}), make(chan struct{}), make(chan struct{})
+ var once sync.Once
+ unblock := func() { once.Do(func() { close(release) }) }
+ defer unblock()
+ m.supplementalProviders = map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider{
+ unifiedresources.SourceTrueNAS: wiringRecordsFunc(func(*Monitor, string) []unifiedresources.IngestRecord {
+ close(entered)
+ <-release
+ return wiringRecords("old-host")
+ }),
+ }
+ go func() { defer close(done); m.updateResourceStore(m.currentStateWithScope()) }()
+ waitForSignal(t, entered, 5*time.Second, "old collector did not start")
+ var replacement MonitorSupplementalRecordsProvider
+ if !withdraw {
+ replacement = wiringRecordsFunc(func(*Monitor, string) []unifiedresources.IngestRecord { return wiringRecords("new-host") })
+ }
+ m.SetSupplementalRecordsProvider(unifiedresources.SourceTrueNAS, replacement)
+ unblock()
+ waitForSignal(t, done, 5*time.Second, "old collector did not return")
+ if len(store.batches) != 1 {
+ t.Fatalf("published %v, want only the changed binding", store.batches)
+ }
+ want := 1
+ if withdraw {
+ want = 0
+ }
+ if len(store.batches[0]) != want || (!withdraw && store.batches[0][0] != "new-host") {
+ t.Fatalf("publications = %v", store.batches)
+ }
+ for _, resource := range store.GetAll() {
+ if resource.Name == "old-host" {
+ t.Fatal("withdrawn/replaced host was republished")
+ }
+ }
+ })
+ }
+}
+
+// A registration that changes during a store write queues its latest publish
+// after that write. It cannot finish and then be overwritten by the old pass.
+func TestResourceWiringOrdersChangesDuringPublication(t *testing.T) {
+ store := newWiringPublicationStore()
+ m := &Monitor{state: models.NewState(), resourceStore: store}
+ m.supplementalProviders = map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider{
+ unifiedresources.SourceTrueNAS: wiringRecordsFunc(func(*Monitor, string) []unifiedresources.IngestRecord { return wiringRecords("old-host") }),
+ }
+ entered, release, oldDone, newDone := make(chan struct{}), make(chan struct{}), make(chan struct{}), make(chan struct{})
+ var once, released sync.Once
+ unblock := func() { released.Do(func() { close(release) }) }
+ defer unblock()
+ store.before = func() { once.Do(func() { close(entered); <-release }) }
+ go func() { defer close(oldDone); m.updateResourceStore(m.currentStateWithScope()) }()
+ waitForSignal(t, entered, 5*time.Second, "old publication did not start")
+ go func() {
+ defer close(newDone)
+ m.SetSupplementalRecordsProvider(unifiedresources.SourceTrueNAS, wiringRecordsFunc(func(*Monitor, string) []unifiedresources.IngestRecord { return wiringRecords("new-host") }))
+ }()
+ waitForCondition(t, 5*time.Second, func() bool { return m.resourceWiringGeneration.Load() != 0 }, "new binding was not registered")
+ select {
+ case <-newDone:
+ t.Fatal("new binding completed before the old store write finished")
+ default:
+ }
+ unblock()
+ waitForSignal(t, oldDone, 5*time.Second, "old publication did not return")
+ waitForSignal(t, newDone, 5*time.Second, "new publication did not return")
+ if len(store.batches) != 2 || store.batches[1][0] != "new-host" {
+ t.Fatalf("publications = %v, want old then new", store.batches)
+ }
+ if !m.mockModeFence.registryCurrent() {
+ t.Fatal("final registry remained marked stale")
+ }
+ for _, resource := range store.GetAll() {
+ if resource.Name == "old-host" {
+ t.Fatal("old row survived the changed binding")
+ }
+ }
+}
+
+func TestResourceWiringProviderCanReenterWithoutPublishingItsOldRecords(t *testing.T) {
+ store := newWiringPublicationStore()
+ m := &Monitor{state: models.NewState(), resourceStore: store}
+ m.supplementalProviders = map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider{
+ unifiedresources.SourceTrueNAS: wiringRecordsFunc(func(m *Monitor, _ string) []unifiedresources.IngestRecord {
+ m.SetSupplementalRecordsProvider(unifiedresources.SourceTrueNAS, wiringRecordsFunc(func(*Monitor, string) []unifiedresources.IngestRecord { return wiringRecords("new-host") }))
+ return wiringRecords("old-host")
+ }),
+ }
+ done := make(chan struct{})
+ go func() { defer close(done); m.updateResourceStore(m.currentStateWithScope()) }()
+ waitForSignal(t, done, 5*time.Second, "provider re-entry deadlocked")
+ if len(store.batches) != 1 || store.batches[0][0] != "new-host" {
+ t.Fatalf("publications = %v", store.batches)
+ }
+}
+
+func TestResourceWiringJoinsAdmittedChecksAndRefusesOldScopes(t *testing.T) {
+ m := &Monitor{state: models.NewState()}
+ old := m.mockModeFence.begin()
+ entered, release, checkDone, wiringDone := make(chan struct{}), make(chan struct{}), make(chan struct{}), make(chan struct{})
+ var once sync.Once
+ unblock := func() { once.Do(func() { close(release) }) }
+ defer unblock()
+ go func() { defer close(checkDone); old.run(func() { close(entered); <-release }) }()
+ waitForSignal(t, entered, 5*time.Second, "old check did not start")
+ go func() {
+ defer close(wiringDone)
+ m.SetSupplementalRecordsProvider(unifiedresources.SourceTrueNAS, storeWiringProvider("new-host"))
+ }()
+ waitForCondition(t, 5*time.Second, func() bool { return m.resourceWiringGeneration.Load() != 0 }, "binding did not change")
+ select {
+ case <-wiringDone:
+ t.Fatal("rewiring did not join admitted old checks")
+ default:
+ }
+ unblock()
+ waitForSignal(t, checkDone, 5*time.Second, "old check did not finish")
+ waitForSignal(t, wiringDone, 5*time.Second, "wiring did not finish")
+ if old.run(func() { t.Error("old check was admitted after rewiring") }) {
+ t.Fatal("old scope remained usable")
+ }
+ if !m.mockModeFence.begin().run(func() {}) {
+ t.Fatal("new scope refused its check")
+ }
+}
+
+// Store changes must invalidate a collector too, even with identical providers.
+func TestResourceWiringRefusesCollectedRecordsForAReplacedStore(t *testing.T) {
+ oldStore, newStore := newWiringPublicationStore(), newWiringPublicationStore()
+ m := &Monitor{state: models.NewState(), resourceStore: oldStore}
+ entered, release, done := make(chan struct{}), make(chan struct{}), make(chan struct{})
+ var held atomic.Bool
+ var once sync.Once
+ unblock := func() { once.Do(func() { close(release) }) }
+ defer unblock()
+ p := wiringRecordsFunc(func(*Monitor, string) []unifiedresources.IngestRecord {
+ if held.CompareAndSwap(false, true) {
+ close(entered)
+ <-release
+ }
+ return wiringRecords("host")
+ })
+ m.supplementalProviders = map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider{unifiedresources.SourceTrueNAS: p}
+ go func() { defer close(done); m.updateResourceStore(m.currentStateWithScope()) }()
+ waitForSignal(t, entered, 5*time.Second, "collector did not start")
+ m.SetResourceStore(newStore)
+ unblock()
+ waitForSignal(t, done, 5*time.Second, "collector did not return")
+ if len(oldStore.batches) != 0 || len(newStore.batches) != 1 {
+ t.Fatalf("old store published %v; new store published %v", oldStore.batches, newStore.batches)
+ }
+}
+
+type modeBoundWiringProvider struct{ mock bool }
+
+func (p *modeBoundWiringProvider) SupplementalRecordsMockMode() bool { return p.mock }
+func (p *modeBoundWiringProvider) SupplementalRecords(*Monitor, string) []unifiedresources.IngestRecord {
+ return wiringRecords("bound-host")
+}
+func (p *modeBoundWiringProvider) SnapshotOwnedSources() []unifiedresources.DataSource {
+ return []unifiedresources.DataSource{unifiedresources.SourceTrueNAS}
+}
+
+// Exercise both intervals before the post-switch router hook has replaced the
+// old pair. Its records and ownership must be absent, not a live/demo mixture.
+func TestResourceWiringModeBoundProvidersDoNotCrossTheSwitchHookGap(t *testing.T) {
+ previous := mock.IsMockEnabled()
+ t.Cleanup(func() { mustSetMockEnabled(t, previous) })
+ for _, mode := range []bool{false, true} {
+ mustSetMockEnabled(t, mode)
+ m := &Monitor{supplementalProviders: map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider{unifiedresources.SourceTrueNAS: &modeBoundWiringProvider{mock: !mode}}}
+ if got := m.collectSupplementalRecordsBySource(); len(got) != 0 {
+ t.Fatalf("mode %v included opposite-mode records: %v", mode, got)
+ }
+ if got := m.providerOwnedSnapshotSources(); len(got) != 0 {
+ t.Fatalf("mode %v kept opposite-mode snapshot ownership: %v", mode, got)
+ }
+ m.supplementalProviders[unifiedresources.SourceTrueNAS] = &modeBoundWiringProvider{mock: mode}
+ if got := m.collectSupplementalRecordsBySource(); len(got[unifiedresources.SourceTrueNAS]) != 1 {
+ t.Fatalf("mode %v lost its matching records: %v", mode, got)
+ }
+ }
+}
+
+func TestResourceWiringDoesNotClearExistingSystemAlerts(t *testing.T) {
+ manager := newMockHostAlertTestManager(t)
+ if !manager.RaiseSystemAlert(alerts.SystemAlertInput{Type: alerts.BackupEvaluationAlertType, Level: alerts.AlertLevelWarning, Message: "existing condition"}) {
+ t.Fatal("control did not create system alert")
+ }
+ m := &Monitor{state: models.NewState(), alertManager: manager}
+ m.SetSupplementalRecordsProvider(unifiedresources.SourceTrueNAS, storeWiringProvider("new-host"))
+ active := manager.GetActiveAlerts()
+ if len(active) != 1 || active[0].Type != alerts.BackupEvaluationAlertType {
+ t.Fatalf("rewiring cleared existing alerts: %v", active)
+ }
+}
+
+func TestResourceWiringSingleProviderPublishesWithoutLegacyState(t *testing.T) {
+ store := newWiringPublicationStore()
+ m := &Monitor{resourceStore: store}
+ m.SetSupplementalRecordsProvider(unifiedresources.SourceTrueNAS, storeWiringProvider("native-host"))
+ if len(store.batches) != 1 || len(store.batches[0]) != 1 || store.batches[0][0] != "native-host" {
+ t.Fatalf("source-native backfill without state = %v", store.batches)
+ }
+}
diff --git a/internal/monitoring/truenas_poller.go b/internal/monitoring/truenas_poller.go
index 9c5f784936..c620be45ff 100644
--- a/internal/monitoring/truenas_poller.go
+++ b/internal/monitoring/truenas_poller.go
@@ -993,6 +993,11 @@ func (p *TrueNASPoller) GetCurrentRecords() []unifiedresources.IngestRecord {
// SupplementalRecords adapts TrueNAS records to the monitor supplemental
// provider contract.
+// SupplementalRecordsMockMode binds native records to real infrastructure.
+// Mock-mode publications must not read this cached inventory before the
+// router's post-switch hook replaces the platform provider pair.
+func (*TrueNASPoller) SupplementalRecordsMockMode() bool { return false }
+
func (p *TrueNASPoller) SupplementalRecords(_ *Monitor, orgID string) []unifiedresources.IngestRecord {
return p.GetCurrentRecordsForOrg(orgID)
}
diff --git a/internal/monitoring/vmware_poller.go b/internal/monitoring/vmware_poller.go
index fc4a7667bd..a2bf7ee945 100644
--- a/internal/monitoring/vmware_poller.go
+++ b/internal/monitoring/vmware_poller.go
@@ -512,6 +512,11 @@ func (p *VMwarePoller) GetCurrentRecords() []unifiedresources.IngestRecord {
}
// SupplementalRecords implements the monitor supplemental-record provider contract.
+// SupplementalRecordsMockMode binds native records to real infrastructure.
+// Mock-mode publications must not read this cached inventory before the
+// router's post-switch hook replaces the platform provider pair.
+func (*VMwarePoller) SupplementalRecordsMockMode() bool { return false }
+
func (p *VMwarePoller) SupplementalRecords(_ *Monitor, orgID string) []unifiedresources.IngestRecord {
return p.GetCurrentRecordsForOrg(orgID)
}