Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
fix(monitoring): publish complete resource wiring once
pulse-triage[bot]da44cec421a1
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex b198549729..18f15e5aa2 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -17,6 +17,40 @@## Purpose+### Repeatable resource wiring publishes a complete binding once++Router and tenant initialization applies saved notification policy before the+first resource publication, then attaches the store and supplemental provider+batch together. Repeating identical wiring does not rebuild the estate; the+single setters retain their explicit-refresh behaviour. Explicit nil provider+withdrawals remain in the router's initializer registry so existing and future+tenants cannot retain a removed source. An empty saved private-CIDR allowlist+revokes previous grants; absent/unreadable settings and the automatic public-URL+fallback retain their existing semantics.++A refresh captures its store/provider generation before collecting records.+Delayed old collectors are refused before publication; full store writes are+ordered after collection, without holding monitor or provider locks across+callbacks. Changed bindings advance the existing alert epoch and join admitted+old checks without clearing active alerts. Dependency setters, like a mock-mode+switch, must not change bindings from an admitted alert callback. Consume-once+source changes are drained only after an admitted publication, outside the+publication lock. The read path keeps its existing non-blocking/freshness gate.++Native TrueNAS/VMware providers and the demo adapter declare their mode. One+mode read filters the pair before records, ownership or changes are collected,+including the interval before the existing post-switch hook replaces them.+Opposite-mode data cannot become a live/demo mixture; tenant fixture exclusions+remain. Runtime-check shutdown, cancellation, token/tenant boundaries, resource+wire shapes, thresholds, retention and guest/TLS safety fences are unchanged.++`resource_wiring_test.go` and `router_resource_wiring_test.go` cover pass counts,+first-alert policy, allowlist revocation, withdrawals, delayed collection and+publication, re-entry, admitted-check joins and switch-hook gaps. These are+bounded synthetic component controls, not whole-source, native startup/RSS,+notification delivery or installed/reporter acceptance.++### Guest filesystem guidance honours its caller's loggerMissing-filesystem guidance uses the existing caller context logger, fallingdiff --git a/internal/api/router.go b/internal/api/router.goindex fe2766d9e5..4d92680657 100644--- a/internal/api/router.go+++ b/internal/api/router.go@@ -112,6 +112,7 @@ type Router struct {monitorResourceAdapter *unifiedresources.MonitorAdaptermonitorResourceAdapters map[string]*unifiedresources.MonitorAdaptermonitorAdapterMu sync.Mutex+ monitorSupplementalMu sync.Mutex // serializes provider updates with complete monitor initializationmonitorSupplementalRecords map[unifiedresources.DataSource]monitoring.MonitorSupplementalRecordsProviderreportingHandlers *ReportingHandlersconfigProfileHandler *ConfigProfileHandler@@ -1882,11 +1883,15 @@ func (r *Router) configureMonitorDependencies(m *monitoring.Monitor) {return}- if adapter := r.monitorAdapterForMonitor(m); adapter != nil {- log.Debug().Msg("[Router] Injecting unified resource adapter into monitor")- m.SetResourceStore(adapter)- }+ r.monitorSupplementalMu.Lock()+ defer r.monitorSupplementalMu.Unlock()+ r.configureMonitorDependenciesLocked(m)+}+// The caller holds monitorSupplementalMu across reading and applying the+// registry, not just copying the map: a delayed initializer must not reinstall+// an old provider after a mock switch or withdrawal has completed.+func (r *Router) configureMonitorDependenciesLocked(m *monitoring.Monitor) {// Tenant monitors must inherit the persisted instance-wide notification// settings. System settings are stored globally, so without this a tenant// monitor created after the allowlist was saved (or after a restart)@@ -1896,11 +1901,11 @@ func (r *Router) configureMonitorDependencies(m *monitoring.Monitor) {if r.persistence != nil {if settings, err := r.persistence.LoadSystemSettings(); err == nil && settings != nil {if nm := m.GetNotificationManager(); nm != nil {- if settings.WebhookAllowedPrivateCIDRs != "" {- if err := nm.UpdateAllowedPrivateCIDRs(settings.WebhookAllowedPrivateCIDRs); err != nil {- log.Error().Err(err).Msg("Failed to apply webhook allowed private CIDRs to tenant monitor")- }+ if err := nm.UpdateAllowedPrivateCIDRs(settings.WebhookAllowedPrivateCIDRs); err != nil {+ log.Error().Err(err).Msg("Failed to apply webhook allowed private CIDRs to tenant monitor")}+ // An empty allowlist revokes old grants. An empty public URL+ // keeps the notification manager's existing automatic fallback.if settings.PublicURL != "" {nm.SetPublicURL(settings.PublicURL)}@@ -1910,46 +1915,45 @@ func (r *Router) configureMonitorDependencies(m *monitoring.Monitor) {}}- if len(r.monitorSupplementalRecords) == 0 {- return- }-- keys := make([]string, 0, len(r.monitorSupplementalRecords))- for source := range r.monitorSupplementalRecords {- keys = append(keys, string(source))- }- sort.Strings(keys)-- for _, key := range keys {- source := unifiedresources.DataSource(key)- provider := r.monitorSupplementalRecords[source]- m.SetSupplementalRecordsProvider(source, provider)+ // Apply saved notification policy before the first publication, which can+ // evaluate alerts. The combined boundary avoids one refresh per source.+ if adapter := r.monitorAdapterForMonitor(m); adapter != nil {+ m.SetResourceStoreAndProviders(adapter, r.monitorSupplementalRecords)+ } else {+ m.SetSupplementalRecordsProviders(r.monitorSupplementalRecords)}}func (r *Router) setMonitorSupplementalRecordsProvider(source unifiedresources.DataSource, provider monitoring.MonitorSupplementalRecordsProvider) {- if r == nil {- return- }+ r.setMonitorSupplementalRecordsProviders(map[unifiedresources.DataSource]monitoring.MonitorSupplementalRecordsProvider{source: provider})+}- normalized := unifiedresources.DataSource(strings.ToLower(strings.TrimSpace(string(source))))- if normalized == "" {+func (r *Router) setMonitorSupplementalRecordsProviders(providers map[unifiedresources.DataSource]monitoring.MonitorSupplementalRecordsProvider) {+ if r == nil {return}-- if r.monitorSupplementalRecords == nil {- r.monitorSupplementalRecords = make(map[unifiedresources.DataSource]monitoring.MonitorSupplementalRecordsProvider)- }- if provider == nil {- delete(r.monitorSupplementalRecords, normalized)- } else {+ r.monitorSupplementalMu.Lock()+ usable := false+ for source, provider := range providers {+ normalized := unifiedresources.DataSource(strings.ToLower(strings.TrimSpace(string(source))))+ if normalized == "" {+ continue+ }+ usable = true+ if r.monitorSupplementalRecords == nil {+ r.monitorSupplementalRecords = make(map[unifiedresources.DataSource]monitoring.MonitorSupplementalRecordsProvider)+ }+ // Retain explicit nil withdrawals in initializer snapshots. Omitting a+ // removed source would leave it installed in already-running tenants.r.monitorSupplementalRecords[normalized] = provider}-- if r.monitor != nil {- r.monitor.SetSupplementalRecordsProvider(normalized, provider)+ if usable && r.monitor != nil {+ r.configureMonitorDependenciesLocked(r.monitor)}- if r.mtMonitor != nil {+ r.monitorSupplementalMu.Unlock()+ if usable && r.mtMonitor != nil {+ // Do not hold the router lock while taking the tenant-manager lock.+ // Its callback obtains a fresh, serialized registry for each monitor.r.mtMonitor.SetMonitorInitializer(r.configureMonitorDependencies)}}@@ -1970,8 +1974,10 @@ func (r *Router) syncPlatformSupplementalProviders(mockEnabled bool) {r.resourceHandlers.SetSupplementalRecordsProvider(unifiedresources.SourceTrueNAS, trueNASAdapter)r.resourceHandlers.SetSupplementalRecordsProvider(unifiedresources.SourceVMware, vmwareAdapter)}- r.setMonitorSupplementalRecordsProvider(unifiedresources.SourceTrueNAS, trueNASAdapter)- r.setMonitorSupplementalRecordsProvider(unifiedresources.SourceVMware, vmwareAdapter)+ r.setMonitorSupplementalRecordsProviders(map[unifiedresources.DataSource]monitoring.MonitorSupplementalRecordsProvider{+ unifiedresources.SourceTrueNAS: trueNASAdapter,+ unifiedresources.SourceVMware: vmwareAdapter,+ })return}@@ -1982,8 +1988,10 @@ func (r *Router) syncPlatformSupplementalProviders(mockEnabled bool) {r.resourceHandlers.SetSupplementalRecordsProvider(unifiedresources.SourceTrueNAS, r.trueNASPoller)r.resourceHandlers.SetSupplementalRecordsProvider(unifiedresources.SourceVMware, r.vmwarePoller)}- r.setMonitorSupplementalRecordsProvider(unifiedresources.SourceTrueNAS, r.trueNASPoller)- r.setMonitorSupplementalRecordsProvider(unifiedresources.SourceVMware, r.vmwarePoller)+ r.setMonitorSupplementalRecordsProviders(map[unifiedresources.DataSource]monitoring.MonitorSupplementalRecordsProvider{+ unifiedresources.SourceTrueNAS: r.trueNASPoller,+ unifiedresources.SourceVMware: r.vmwarePoller,+ })}// getTenantMonitor returns the appropriate monitor for the current request's tenant.@@ -8442,6 +8450,8 @@ func (a mockSupplementalRecordsAdapter) GetCurrentRecordsForOrg(orgID string) []return mock.SupplementalRecords(a.source)}+func (mockSupplementalRecordsAdapter) SupplementalRecordsMockMode() bool { return true }+func (a mockSupplementalRecordsAdapter) SupplementalRecords(_ *monitoring.Monitor, orgID string) []unifiedresources.IngestRecord {return a.GetCurrentRecordsForOrg(orgID)}diff --git a/internal/api/router_resource_wiring_test.go b/internal/api/router_resource_wiring_test.gonew file mode 100644index 0000000000..00245a0676--- /dev/null+++ b/internal/api/router_resource_wiring_test.go@@ -0,0 +1,295 @@+package api++import (+ "github.com/rcourtman/pulse-go-rewrite/internal/alerts"+ "github.com/rcourtman/pulse-go-rewrite/internal/config"+ "github.com/rcourtman/pulse-go-rewrite/internal/models"+ "github.com/rcourtman/pulse-go-rewrite/internal/monitoring"+ "github.com/rcourtman/pulse-go-rewrite/internal/truenas"+ "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"+ "github.com/rcourtman/pulse-go-rewrite/internal/vmware"+ "sync"+ "sync/atomic"+ "testing"+ "time"+)++type countingSupplementalRecords struct {+ reads atomic.Int32+}++func (c *countingSupplementalRecords) SupplementalRecords(*monitoring.Monitor, string) []unifiedresources.IngestRecord {+ c.reads.Add(1)+ return nil+}++func newWiringTestMonitor(t *testing.T) *monitoring.Monitor {+ t.Helper()+ monitor, err := monitoring.New(&config.Config{DataPath: t.TempDir()})+ if err != nil {+ t.Fatalf("new monitor: %v", err)+ }+ t.Cleanup(func() { monitor.Stop() })+ return monitor+}++// Wiring a monitor attaches its store and every supplemental provider. Each+// separate wiring step published the whole estate, so a router with two+// platform providers ran three full store passes (and the first two read an+// incomplete provider set) before the listener opened.+func TestConfigureMonitorDependenciesPublishesTheEstateOnce(t *testing.T) {+ trueNAS := &countingSupplementalRecords{}+ vmware := &countingSupplementalRecords{}+ router := &Router{+ monitorResourceAdapter: unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil)),+ monitorResourceAdapters: make(map[string]*unifiedresources.MonitorAdapter),+ monitorSupplementalRecords: map[unifiedresources.DataSource]monitoring.MonitorSupplementalRecordsProvider{+ unifiedresources.SourceTrueNAS: trueNAS,+ unifiedresources.SourceVMware: vmware,+ },+ }++ monitor := newWiringTestMonitor(t)+ router.configureMonitorDependencies(monitor)++ if got := trueNAS.reads.Load(); got != 1 {+ t.Fatalf("TrueNAS provider read %d times while wiring one monitor, want 1 store pass", got)+ }+ if got := vmware.reads.Load(); got != 1 {+ t.Fatalf("VMware provider read %d times while wiring one monitor, want 1 store pass", got)+ }++ // NewRouter, SetMonitor, SetMultiTenantMonitor and every provider change+ // wire the same monitor again; none of it has anything new to publish.+ router.configureMonitorDependencies(monitor)+ router.configureMonitorDependencies(monitor)+ if trueNAS.reads.Load() != 1 || vmware.reads.Load() != 1 {+ t.Fatalf("re-wiring the same monitor read the providers %d and %d times in all, want one store pass", trueNAS.reads.Load(), vmware.reads.Load())+ }+}++// passCountingStore counts the registry rebuilds a monitor publishes into it.+type passCountingStore struct {+ *unifiedresources.MonitorAdapter+ passes atomic.Int32+}++func (s *passCountingStore) PopulateSnapshotAndSupplemental(snapshot models.StateSnapshot, recordsBySource map[unifiedresources.DataSource][]unifiedresources.IngestRecord) {+ s.passes.Add(1)+ s.MonitorAdapter.PopulateSnapshotAndSupplemental(snapshot, recordsBySource)+}++// A mock-mode switch re-registers both platform providers; the running monitor+// must publish once for the pair, not once per source.+func TestSyncPlatformSupplementalProvidersPublishesOnceForBothSources(t *testing.T) {+ previousTrueNAS := truenas.IsFeatureEnabled()+ previousVMware := vmware.IsFeatureEnabled()+ t.Cleanup(func() {+ truenas.SetFeatureEnabled(previousTrueNAS)+ vmware.SetFeatureEnabled(previousVMware)+ })++ monitor := newWiringTestMonitor(t)+ store := &passCountingStore{MonitorAdapter: unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil))}+ monitor.SetResourceStore(store)+ router := &Router{monitor: monitor}++ before := store.passes.Load()+ router.syncPlatformSupplementalProviders(true)+ if got := store.passes.Load() - before; got != 1 {+ t.Fatalf("switching mock mode on published %d times, want 1 for the TrueNAS and VMware pair", got)+ }++ before = store.passes.Load()+ router.syncPlatformSupplementalProviders(false)+ if got := store.passes.Load() - before; got != 1 {+ t.Fatalf("switching mock mode off published %d times, want 1 for the TrueNAS and VMware pair", got)+ }+}++// This uses a real alert on the first publication, not a later provider read:+// saved policy must be installed before any alert reaches the lifecycle hook.+func TestConfigureMonitorDependenciesAppliesPolicyBeforeFirstAlert(t *testing.T) {+ monitor := newWiringTestMonitor(t)+ manager := monitor.GetAlertManager()+ cfg := manager.GetConfig()+ cfg.Enabled = false+ manager.UpdateConfig(cfg)+ // Install a cached source-native provider before a store is attached. Its+ // first publication evaluates a real TrueNAS memory threshold; a normal+ // host-agent row is checked on its separate legacy alert path instead.+ provider := &alertWiringProvider{}+ monitor.SetSupplementalRecordsProvider(unifiedresources.SourceTrueNAS, provider)+ cfg = manager.GetConfig()+ cfg.Enabled = true+ cfg.ActivationState = alerts.ActivationActive+ cfg.TimeThresholds = map[string]int{}+ cfg.MetricTimeThresholds = nil+ cfg.TrueNASDefaults.Memory = &alerts.HysteresisThreshold{Trigger: 1, Clear: 0.5}+ manager.UpdateConfig(cfg)+ persistence := config.NewConfigPersistence(t.TempDir())+ settings := config.DefaultSystemSettings()+ settings.PublicURL = "https://pulse.example.test"+ settings.WebhookAllowedPrivateCIDRs = "192.168.50.0/24"+ if err := persistence.SaveSystemSettings(*settings); err != nil {+ t.Fatal(err)+ }+ notifications := monitor.GetNotificationManager()+ if notifications.ValidateWebhookURL("http://192.168.50.2/hook") == nil {+ t.Fatal("fresh manager already allows the saved private network")+ }+ observations := 0+ unsubscribe := manager.SubscribeLifecycleCallback(func(alerts.LifecycleEvent) {+ observations+++ if got := notifications.GetPublicURL(); got != settings.PublicURL {+ t.Errorf("first-publication alert used PublicURL %q, want %q", got, settings.PublicURL)+ }+ if err := notifications.ValidateWebhookURL("http://192.168.50.2/hook"); err != nil {+ t.Errorf("saved allowlist absent at first-publication alert: %v", err)+ }+ if notifications.ValidateWebhookURL("http://192.168.51.2/hook") == nil {+ t.Error("unlisted private network was admitted")+ }+ })+ defer unsubscribe()+ router := &Router{persistence: persistence, monitorResourceAdapter: unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil))}+ router.configureMonitorDependencies(monitor)+ if observations == 0 {+ resources := wiringResources(monitor)+ if len(resources) != 1 {+ t.Fatalf("no alert; resources = %v", resources)+ }+ input, _ := alerts.UnifiedResourceInputFromResource(resources[0])+ t.Fatalf("first publication emitted no lifecycle event; input=%+v policy=%+v", input, manager.GetConfig().AgentDefaults)+ }+}++type tenantWiringProvider struct{ prefix string }++func (p *tenantWiringProvider) SupplementalRecords(_ *monitoring.Monitor, org string) []unifiedresources.IngestRecord {+ name := p.prefix + "-" + org+ return []unifiedresources.IngestRecord{{SourceID: name, Resource: unifiedresources.Resource{Type: unifiedresources.ResourceTypeAgent, Name: name, Status: unifiedresources.StatusOnline, LastSeen: time.Now().UTC()}, Identity: unifiedresources.ResourceIdentity{Hostnames: []string{name}}}}+}++func TestRouterResourceWiringWithdrawsProvidersFromEveryExistingAndFutureTenant(t *testing.T) {+ persistence := config.NewMultiTenantPersistence(t.TempDir())+ for _, org := range []string{"tenant-a", "tenant-b", "tenant-c"} {+ if err := persistence.SaveOrganization(&models.Organization{ID: org}); err != nil {+ t.Fatal(err)+ }+ }+ mt := monitoring.NewMultiTenantMonitor(&config.Config{DataPath: t.TempDir()}, persistence, nil)+ defer mt.Stop()+ router := &Router{mtMonitor: mt}+ provider := &tenantWiringProvider{prefix: "source"}+ router.setMonitorSupplementalRecordsProvider(unifiedresources.SourceTrueNAS, provider)+ for _, org := range []string{"tenant-a", "tenant-b"} {+ m, err := mt.GetMonitor(org)+ if err != nil {+ t.Fatal(err)+ }+ resources := wiringResources(m)+ if len(resources) != 1 || resources[0].Name != "source-"+org {+ t.Fatalf("%s registry = %v", org, resources)+ }+ }+ router.setMonitorSupplementalRecordsProvider(unifiedresources.SourceTrueNAS, nil)+ for _, org := range []string{"tenant-a", "tenant-b", "tenant-c"} {+ m, err := mt.GetMonitor(org)+ if err != nil {+ t.Fatal(err)+ }+ if resources := wiringResources(m); len(resources) != 0 {+ t.Fatalf("%s retained withdrawn provider: %v", org, resources)+ }+ }+}++func TestRouterResourceWiringSerializesInitializationAndWithdrawal(t *testing.T) {+ monitor := newWiringTestMonitor(t)+ entered, release, initialized, withdrawn := make(chan struct{}), make(chan struct{}), make(chan struct{}), make(chan struct{})+ var once sync.Once+ unblock := func() { once.Do(func() { close(release) }) }+ defer unblock()+ provider := &blockingRouterWiringProvider{entered: entered, release: release}+ router := &Router{monitor: monitor, monitorResourceAdapter: unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil)), monitorSupplementalRecords: map[unifiedresources.DataSource]monitoring.MonitorSupplementalRecordsProvider{unifiedresources.SourceTrueNAS: provider}}+ go func() { defer close(initialized); router.configureMonitorDependencies(monitor) }()+ select {+ case <-entered:+ case <-time.After(5 * time.Second):+ t.Fatal("initializer did not collect")+ }+ go func() {+ defer close(withdrawn)+ router.setMonitorSupplementalRecordsProvider(unifiedresources.SourceTrueNAS, nil)+ }()+ // Check the exact exclusion boundary while the initializer is blocked.+ if router.monitorSupplementalMu.TryLock() {+ router.monitorSupplementalMu.Unlock()+ t.Fatal("initializer released its provider registry before applying it")+ }+ unblock()+ select {+ case <-initialized:+ case <-time.After(5 * time.Second):+ t.Fatal("initializer did not finish")+ }+ select {+ case <-withdrawn:+ case <-time.After(5 * time.Second):+ t.Fatal("withdrawal did not finish")+ }+ if resources := wiringResources(monitor); len(resources) != 0 {+ t.Fatalf("withdrawal left rows: %v", resources)+ }+}++type blockingRouterWiringProvider struct{ entered, release chan struct{} }++func (p *blockingRouterWiringProvider) SupplementalRecords(*monitoring.Monitor, string) []unifiedresources.IngestRecord {+ close(p.entered)+ <-p.release+ return (&tenantWiringProvider{prefix: "old"}).SupplementalRecords(nil, "default")+}++func wiringResources(m *monitoring.Monitor) []unifiedresources.Resource {+ return m.GetUnifiedReadState().(interface {+ GetAll() []unifiedresources.Resource+ }).GetAll()+}++func TestConfigureMonitorDependenciesRevokesEmptySavedAllowlist(t *testing.T) {+ monitor := newWiringTestMonitor(t)+ persistence := config.NewConfigPersistence(t.TempDir())+ settings := config.DefaultSystemSettings()+ settings.PublicURL = "https://pulse.example.test"+ settings.WebhookAllowedPrivateCIDRs = "192.168.50.0/24"+ if err := persistence.SaveSystemSettings(*settings); err != nil {+ t.Fatal(err)+ }+ router := &Router{persistence: persistence, monitorResourceAdapter: unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil))}+ router.configureMonitorDependencies(monitor)+ nm := monitor.GetNotificationManager()+ if nm.GetPublicURL() != settings.PublicURL || nm.ValidateWebhookURL("http://192.168.50.2/hook") != nil {+ t.Fatal("saved policy did not apply")+ }+ settings.WebhookAllowedPrivateCIDRs = ""+ if err := persistence.SaveSystemSettings(*settings); err != nil {+ t.Fatal(err)+ }+ router.configureMonitorDependencies(monitor)+ if nm.GetPublicURL() != settings.PublicURL || nm.ValidateWebhookURL("http://192.168.50.2/hook") == nil {+ t.Fatal("reinitialization retained revoked notification policy")+ }+}++// A cached, synthetic source-native observation. No provider request is made.+type alertWiringProvider struct{}++func (*alertWiringProvider) SupplementalRecords(*monitoring.Monitor, string) []unifiedresources.IngestRecord {+ return []unifiedresources.IngestRecord{{SourceID: "policy-proof-host", Resource: unifiedresources.Resource{+ Type: unifiedresources.ResourceTypeAgent, Name: "policy-proof-host", Status: unifiedresources.StatusOnline, LastSeen: time.Now().UTC(),+ Sources: []unifiedresources.DataSource{unifiedresources.SourceTrueNAS},+ Metrics: &unifiedresources.ResourceMetrics{Memory: &unifiedresources.MetricValue{Value: 99, Percent: 99, Unit: "percent"}},+ }, Identity: unifiedresources.ResourceIdentity{Hostnames: []string{"policy-proof-host"}}}}+}diff --git a/internal/monitoring/mock_mode_fence.go b/internal/monitoring/mock_mode_fence.goindex c552480bbe..9699a275d5 100644--- a/internal/monitoring/mock_mode_fence.go+++ b/internal/monitoring/mock_mode_fence.go@@ -5,6 +5,11 @@ import ("sync/atomic")+// Store/provider rewiring also advances this epoch: its old inventory must+// not reach alerts after the new dependency binding is published. Unlike a+// mode switch, rewiring does not clear active alerts. Both callers join only+// admitted alert-manager calls, never a slow provider read or registry write.+//// mockModeFence keeps alert evaluations of mode-dependent data inside the// mock-mode epoch they read it in. GetState, the fixture graph, recovery// rollups and the connection ledger all switch source when mock mode flips,diff --git a/internal/monitoring/monitor.go b/internal/monitoring/monitor.goindex 6e76f913f9..9977be5ec5 100644--- a/internal/monitoring/monitor.go+++ b/internal/monitoring/monitor.go@@ -1426,6 +1426,9 @@ type Monitor struct {nodePendingUpdatesCache map[string]pendingUpdatesCache // Cache pending updates per node (checked every 30 min)pveMembershipMisses map[string]int // Consecutive authoritative membership absences by instance/noderesourceStore ResourceStoreInterface // Optional unified resource store for polling optimization+ resourceWiringMu sync.Mutex // Serializes dependency changes and their attachment effects, not provider/alert callbacks+ resourceWiringGeneration atomic.Uint64 // Store/provider binding captured by a refresh+ resourcePublishMu sync.Mutex // Serializes full publications after provider collectionsupplementalProviders map[unifiedresources.DataSource]MonitorSupplementalRecordsProviderrecoveryManager *recoverymanager.Manager // Optional recovery store manager for backup rollupsrecoveryIngestMu sync.Mutex@@ -5028,73 +5031,147 @@ func (m *Monitor) DisableTemperatureMonitoring() {log.Info().Msg("temperature monitoring disabled")}-// SetResourceStore sets the resource store for polling optimization.-// When set, the monitor will check if it should reduce polling frequency-// for nodes that have host agents providing data.+// SetResourceStore sets the resource store and immediately backfills it, even+// when the same store is supplied again. Use SetResourceStoreAndProviders for+// repeatable router/tenant initialization without redundant full refreshes.func (m *Monitor) SetResourceStore(store ResourceStoreInterface) {+ m.wireResourceStoreAndProviders(store, nil, true)+}++// SetResourceStoreAndProviders attaches the store and a batch of provider+// updates in one critical section, then publishes once if anything changed.+// Omitted sources are unchanged; an explicit nil withdraws a source. Like a+// mock-mode switch, changing dependencies must not run inside an admitted+// alert callback: it joins old alert calls before the new binding can publish.+func (m *Monitor) SetResourceStoreAndProviders(store ResourceStoreInterface, providers map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider) {+ m.wireResourceStoreAndProviders(store, providers, false)+}++// sameWiredValue treats uncomparable dependency values as changed, never as a+// reason to panic or silently skip a refresh.+func sameWiredValue(a, b any) (same bool) {+ defer func() {+ if recover() != nil {+ same = false+ }+ }()+ return a == b+}++func (m *Monitor) wireResourceStoreAndProviders(store ResourceStoreInterface, providers map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider, force bool) {+ if m == nil {+ return+ }+ m.resourceWiringMu.Lock()if thresholdStore, ok := store.(StaleThresholdResourceStore); ok {thresholdStore.SetStaleThresholds(m.resourceStaleThresholds())}m.mu.Lock()+ _, providersChanged := m.registerSupplementalProvidersLocked(providers)+ storeChanged := !sameWiredValue(m.resourceStore, store)m.resourceStore = storeincidentStore := m.incidentStorem.installNodeAgentSplitDeciderLocked()+ changed := storeChanged || providersChanged+ if changed {+ m.resourceWiringGeneration.Add(1)+ }m.mu.Unlock()- m.installOperatorIntentResolver(store)- log.Info().Msg("resource store set for polling optimization")- timelineAttached := false- if incidentStore != nil {- if timelineStore, ok := store.(memory.IncidentTimelineStore); ok {+ if storeChanged || force {+ m.installOperatorIntentResolver(store)+ if incidentStore != nil {+ timelineStore, _ := store.(memory.IncidentTimelineStore)incidentStore.SetResourceTimelineStore(timelineStore)- timelineAttached = true- } else {- incidentStore.SetResourceTimelineStore(nil)+ if timelineStore != nil {+ // Router/health serving must not wait for backlog projection.+ m.scheduleAlertProjectionCatchUp()+ }}}- if timelineAttached {- // The canonical boundary: with the durable resource store attached,- // restart repair can reach resource history as well as the incident- // fallback cache, and the replay watermark may advance. Run it in the- // background — router construction and health serving must not wait- // behind an un-projected backlog.- m.scheduleAlertProjectionCatchUp()+ m.resourceWiringMu.Unlock()++ if changed {+ // The epoch guards dependency-derived alerts as well as mode-derived+ // ones. No active alert is cleared here; the new inventory evaluates+ // them normally. No monitor/state lock is held while joining old calls.+ m.mockModeFence.advance()+ }++ if changed || force {+ m.backfillResourceStore(false)}+}- // Immediately backfill the store from current state so ReadState- // consumers have data as soon as the store is wired.- // Guard against minimally initialized monitors (e.g., test fixtures- // with bare &Monitor{}) where m.state may be nil.- if store != nil && m.state != nil {+func (m *Monitor) backfillResourceStore(allowEmptyState bool) {+ // Bare monitor fixtures have no state. Read the pointer under the same+ // lock used by a mock switch replacing it.+ m.mu.RLock()+ ready := m.resourceStore != nil && (m.state != nil || allowEmptyState)+ m.mu.RUnlock()+ if ready {m.updateResourceStore(m.currentStateWithScope())}}-// SetSupplementalRecordsProvider configures source-native resource providers-// that ingest alongside the legacy state snapshot path.+// SetSupplementalRecordsProvider preserves the single-source explicit-refresh+// behavior, including when the supplied provider is unchanged.func (m *Monitor) SetSupplementalRecordsProvider(source unifiedresources.DataSource, provider MonitorSupplementalRecordsProvider) {- if m == nil {- return- }+ m.SetSupplementalRecordsProviders(map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider{source: provider})+}- normalized := unifiedresources.DataSource(strings.ToLower(strings.TrimSpace(string(source))))- if normalized == "" {+// SetSupplementalRecordsProviders applies a batch of source updates and+// refreshes once. Empty source names are ignored; nil withdraws a provider.+// Changed bindings have the same alert-callback restriction as the combined setter.+func (m *Monitor) SetSupplementalRecordsProviders(providers map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider) {+ if m == nil {return}-+ m.resourceWiringMu.Lock()m.mu.Lock()- if m.supplementalProviders == nil {- m.supplementalProviders = make(map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider)- }- if provider == nil {- delete(m.supplementalProviders, normalized)- } else {- m.supplementalProviders[normalized] = provider+ usable, changed := m.registerSupplementalProvidersLocked(providers)+ if changed {+ m.resourceWiringGeneration.Add(1)}m.mu.Unlock()+ m.resourceWiringMu.Unlock()+ if changed {+ m.mockModeFence.advance()+ }+ if usable {+ // Source-native providers can publish into a store even before a+ // legacy state exists (including minimally initialized monitors).+ m.backfillResourceStore(true)+ }+}- m.updateResourceStore(m.currentStateWithScope())+// The caller holds m.mu. Compare and update every source in the same section+// as the store binding, so a refresh can never capture half a wiring batch.+func (m *Monitor) registerSupplementalProvidersLocked(providers map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider) (usable, changed bool) {+ for source, provider := range providers {+ normalized := unifiedresources.DataSource(strings.ToLower(strings.TrimSpace(string(source))))+ if normalized == "" {+ continue+ }+ usable = true+ previous, registered := m.supplementalProviders[normalized]+ if provider == nil {+ if registered {+ delete(m.supplementalProviders, normalized)+ changed = true+ }+ continue+ }+ if !registered || !sameWiredValue(previous, provider) {+ changed = true+ }+ if m.supplementalProviders == nil {+ m.supplementalProviders = make(map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider)+ }+ m.supplementalProviders[normalized] = provider+ }+ return usable, changed}// SetLicenseChecker wires the commercial feature gate used by monitoring-owned@@ -5677,6 +5754,7 @@ func (m *Monitor) currentStateWithScope() (models.StateSnapshot, mockModeScope)func (m *Monitor) updateResourceStore(state models.StateSnapshot, scope mockModeScope) {m.mu.RLock()store := m.resourceStore+ generation := m.resourceWiringGeneration.Load()m.mu.RUnlock()if store == nil {@@ -5709,12 +5787,20 @@ func (m *Monitor) updateResourceStore(state models.StateSnapshot, scope mockModeMsg("[Resources] Suppressing legacy snapshot slices for provider-owned sources")}+ // Provider callbacks can read or re-enter the monitor. Collect without any+ // publication/wiring lock, then refuse a binding that changed meanwhile.+ if !scope.current() {+ return+ }+ recordsBySource := m.collectSupplementalRecordsBySource()+ m.resourcePublishMu.Lock()+ if m.resourceWiringGeneration.Load() != generation {+ m.resourcePublishMu.Unlock()+ return+ }mark, published := scope.publish(func() bool {- recordsBySource := m.collectSupplementalRecordsBySource()- supplementalChanges := m.collectSupplementalChanges()if atomicStore, ok := store.(AtomicSnapshotResourceStore); ok {atomicStore.PopulateSnapshotAndSupplemental(snapshotForStore, recordsBySource)- recordSupplementalResourceChanges(store, supplementalChanges)for source, records := range recordsBySource {if len(records) == 0 {continue@@ -5743,14 +5829,20 @@ func (m *Monitor) updateResourceStore(state models.StateSnapshot, scope mockMode}}- recordSupplementalResourceChanges(store, supplementalChanges)return true})- if !published {+ m.resourcePublishMu.Unlock()+ if !published || m.resourceWiringGeneration.Load() != generation {// The snapshot belongs to the mode the monitor just left.return}- m.syncPublishedResourceStore(store, scope, mark)+ // Drain consume-once changes only for a pass that actually published. No+ // provider callback runs under resourcePublishMu, and changes already+ // drained remain history even if a subsequent rewire replaces the binding.+ recordSupplementalResourceChanges(store, m.collectSupplementalChanges())+ if m.resourceWiringGeneration.Load() == generation {+ m.syncPublishedResourceStore(store, scope, mark)+ }}// resourceSnapshotStore serves one GetAll clone to every consumer of a singlediff --git a/internal/monitoring/monitor_mock_alerts_test.go b/internal/monitoring/monitor_mock_alerts_test.goindex 4b54aa19e5..6029a8726e 100644--- a/internal/monitoring/monitor_mock_alerts_test.go+++ b/internal/monitoring/monitor_mock_alerts_test.go@@ -694,7 +694,7 @@ func alertTypes(active []alerts.Alert) map[string]int {return types}-func TestLiveRefreshIgnoresARegistryRepublishedFromMockMode(t *testing.T) {+func TestLiveRefreshRejectsCollectedRecordsFromMockMode(t *testing.T) {setMockSamplerTestEnv(t, time.Hour, 5*time.Minute)pinDefaultMockEstate(t)manager := newMockHostAlertTestManager(t)@@ -758,7 +758,8 @@ func TestLiveRefreshIgnoresARegistryRepublishedFromMockMode(t *testing.T) {}}- // The mock-mode refresh now publishes the fixture estate over it.+ // The old collector finishes, but must be refused before publication.+ // Collection no longer owns a publication admission spanning the switch.close(provider.release)releasedProvider = truewaitForSignal(t, staleDone, 30*time.Second, "mock-mode refresh did not finish after it was released")@@ -766,20 +767,17 @@ func TestLiveRefreshIgnoresARegistryRepublishedFromMockMode(t *testing.T) {releasedList = truewaitForSignal(t, liveDone, 30*time.Second, "live refresh did not finish after it was released")- if active := manager.GetActiveAlerts(); len(active) != 0 {- t.Fatalf("a live refresh evaluated a registry the mock-mode refresh republished, leaving %d fixture alerts: %s",- len(active), describeAlerts(active))- }- if monitor.mockModeFence.registryCurrent() {- t.Fatal("the registry the mock-mode refresh republished counted as current")+ for _, resource := range store.MonitorAdapter.GetAll() {+ if slices.Contains(resource.Sources, unifiedresources.SourceVMware) {+ t.Fatalf("the ended collector republished vSphere fixture resource %s", resource.ID)+ }}-- // The next live refresh is undisturbed: it replaces the registry and- // evaluates the live records.- monitor.updateResourceStore(monitor.currentStateWithScope())if !monitor.mockModeFence.registryCurrent() {- t.Fatal("an undisturbed live rebuild did not make the registry current")+ t.Fatal("the refused collector disturbed the current live publication")}++ // The admitted live pass still evaluates its real source-native records;+ // refusal of old collections must not become blanket alert suppression.types := alertTypes(manager.GetActiveAlerts())if types["zfs-pool-state"] == 0 {t.Fatalf("the undisturbed live refresh raised no alert from the live TrueNAS records: %v", types)diff --git a/internal/monitoring/poll_providers.go b/internal/monitoring/poll_providers.goindex 212e8a2b12..ed7d2fd1c3 100644--- a/internal/monitoring/poll_providers.go+++ b/internal/monitoring/poll_providers.go@@ -6,6 +6,7 @@ import ("strings""time"+ "github.com/rcourtman/pulse-go-rewrite/internal/mock""github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources""github.com/rcourtman/pulse-go-rewrite/pkg/pbs""github.com/rcourtman/pulse-go-rewrite/pkg/pmg"@@ -913,6 +914,14 @@ func (m *Monitor) supplementalProviderSnapshot() map[unifiedresources.DataSource}m.mu.RUnlock()+ // The flag flips before the router's post-switch hook runs. Use one mode+ // for the pair, and keep these optional callbacks outside the monitor lock.+ mode := mock.IsMockEnabled()+ for source, provider := range snapshot {+ if bound, ok := provider.(interface{ SupplementalRecordsMockMode() bool }); ok && bound.SupplementalRecordsMockMode() != mode {+ delete(snapshot, source)+ }+ }if len(snapshot) == 0 {return nil}diff --git a/internal/monitoring/resource_wiring_test.go b/internal/monitoring/resource_wiring_test.gonew file mode 100644index 0000000000..eff983e2e6--- /dev/null+++ b/internal/monitoring/resource_wiring_test.go@@ -0,0 +1,458 @@+package monitoring++import (+ "github.com/rcourtman/pulse-go-rewrite/internal/alerts"+ "github.com/rcourtman/pulse-go-rewrite/internal/mock"+ "github.com/rcourtman/pulse-go-rewrite/internal/models"+ "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"+ "sync"+ "sync/atomic"+ "testing"+ "time"+)++func storeWiringProvider(sourceID string) *testMonitorSupplementalProvider {+ return &testMonitorSupplementalProvider{+ recordsByOrg: map[string][]unifiedresources.IngestRecord{+ "default": {{+ SourceID: sourceID,+ Resource: unifiedresources.Resource{+ Type: unifiedresources.ResourceTypeAgent,+ Name: sourceID,+ Status: unifiedresources.StatusOnline,+ LastSeen: time.Now().UTC(),+ },+ Identity: unifiedresources.ResourceIdentity{Hostnames: []string{sourceID}},+ }},+ },+ }+}++// Every wiring call that publishes populates the whole registry and runs the+// alert pass over every resource, so startup, which wires a store and one+// provider per supplemental source, must publish once, not once per source.+func TestSetResourceStoreAndProvidersPublishesOnce(t *testing.T) {+ store := &testSupplementalResourceStore{}+ monitor := &Monitor{state: models.NewState()}++ monitor.SetResourceStoreAndProviders(store, map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider{+ unifiedresources.SourceTrueNAS: storeWiringProvider("tn-host-1"),+ unifiedresources.SourceVMware: storeWiringProvider("esxi-host-1"),+ })++ if store.snapshotCalls != 1 {+ t.Fatalf("wiring a store and two providers populated the registry %d times, want 1", store.snapshotCalls)+ }+ for source, sourceID := range map[unifiedresources.DataSource]string{+ unifiedresources.SourceTrueNAS: "tn-host-1",+ unifiedresources.SourceVMware: "esxi-host-1",+ } {+ records := store.recordsBySource[source]+ if len(records) != 1 || records[0].SourceID != sourceID {+ t.Fatalf("the one pass ingested %v for %s, want the %s record", records, source, sourceID)+ }+ }+}++// A router re-applies its wiring to a monitor it already wired (a provider+// change re-runs the tenant initializer on every existing monitor, and so do+// SetMonitor and SetMultiTenantMonitor). The same store and providers must not+// repopulate the registry again; a different provider or store must.+func TestSetResourceStoreAndProvidersPublishesOnlyWhenWiringChanged(t *testing.T) {+ store := &testSupplementalResourceStore{}+ monitor := &Monitor{state: models.NewState()}+ trueNAS := storeWiringProvider("tn-host-1")+ vmware := storeWiringProvider("esxi-host-1")+ providers := map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider{+ unifiedresources.SourceTrueNAS: trueNAS,+ unifiedresources.SourceVMware: vmware,+ }++ monitor.SetResourceStoreAndProviders(store, providers)+ monitor.SetResourceStoreAndProviders(store, providers)+ monitor.SetResourceStoreAndProviders(store, map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider{unifiedresources.SourceVMware: vmware})+ if store.snapshotCalls != 1 {+ t.Fatalf("re-wiring the same store and providers populated the registry %d times in all, want 1", store.snapshotCalls)+ }++ monitor.SetResourceStoreAndProviders(store, map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider{+ unifiedresources.SourceVMware: storeWiringProvider("esxi-host-2"),+ })+ if store.snapshotCalls != 2 {+ t.Fatalf("wiring a different provider populated the registry %d times in all, want 2", store.snapshotCalls)+ }++ monitor.SetResourceStoreAndProviders(&testSupplementalResourceStore{}, nil)+ if store.snapshotCalls != 2 {+ t.Fatalf("the replaced store was populated again: %d calls", store.snapshotCalls)+ }+ replacement, ok := monitor.resourceStore.(*testSupplementalResourceStore)+ if !ok || replacement == store {+ t.Fatalf("the monitor kept the store it was wired with: %#v", monitor.resourceStore)+ }+ if replacement.snapshotCalls != 1 {+ t.Fatalf("wiring a different store populated it %d times, want 1", replacement.snapshotCalls)+ }+}++// A provider type that cannot be compared must read as a change, not panic.+type uncomparableWiringProvider struct {+ records []unifiedresources.IngestRecord+}++func (uncomparableWiringProvider) SupplementalRecords(*Monitor, string) []unifiedresources.IngestRecord {+ return nil+}++func TestSetResourceStoreAndProvidersTreatsUncomparableProvidersAsChanged(t *testing.T) {+ store := &testSupplementalResourceStore{}+ monitor := &Monitor{state: models.NewState()}+ providers := map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider{+ unifiedresources.SourceTrueNAS: uncomparableWiringProvider{},+ }++ monitor.SetResourceStoreAndProviders(store, providers)+ monitor.SetResourceStoreAndProviders(store, providers)++ if store.snapshotCalls != 2 {+ t.Fatalf("an uncomparable provider populated the registry %d times for two wirings, want 2", store.snapshotCalls)+ }+}++func TestSetSupplementalRecordsProvidersPublishesOnce(t *testing.T) {+ store := &testSupplementalResourceStore{}+ monitor := &Monitor{state: models.NewState(), resourceStore: store}++ monitor.SetSupplementalRecordsProviders(map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider{+ unifiedresources.SourceTrueNAS: storeWiringProvider("tn-host-1"),+ unifiedresources.SourceVMware: storeWiringProvider("esxi-host-1"),+ })++ if store.snapshotCalls != 1 {+ t.Fatalf("registering two providers populated the registry %d times, want 1", store.snapshotCalls)+ }+ if len(store.recordsBySource[unifiedresources.SourceTrueNAS]) != 1 || len(store.recordsBySource[unifiedresources.SourceVMware]) != 1 {+ t.Fatalf("the pass did not ingest both providers: %v", store.recordsBySource)+ }+}++func TestSetSupplementalRecordsProvidersNormalizesRemovesAndSkipsEmpty(t *testing.T) {+ store := &testSupplementalResourceStore{}+ monitor := &Monitor{state: models.NewState(), resourceStore: store}++ monitor.SetSupplementalRecordsProviders(map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider{+ " TrueNAS ": storeWiringProvider("tn-host-1"),+ " ": storeWiringProvider("nameless"),+ })+ if store.snapshotCalls != 1 {+ t.Fatalf("a usable source populated the registry %d times, want 1", store.snapshotCalls)+ }+ if len(store.recordsBySource[unifiedresources.SourceTrueNAS]) != 1 {+ t.Fatalf("a provider registered as %q was not found under %q: %v", " TrueNAS ", unifiedresources.SourceTrueNAS, store.recordsBySource)+ }+ if len(store.recordsBySource) != 1 {+ t.Fatalf("a blank source was registered: %v", store.recordsBySource)+ }++ store.snapshotCalls = 0+ monitor.SetSupplementalRecordsProviders(map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider{" ": storeWiringProvider("nameless")})+ monitor.SetSupplementalRecordsProviders(nil)+ if store.snapshotCalls != 0 {+ t.Fatalf("a change naming no usable source populated the registry %d times, want 0", store.snapshotCalls)+ }++ store.recordsBySource = nil+ monitor.SetSupplementalRecordsProviders(map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider{unifiedresources.SourceTrueNAS: nil})+ if store.snapshotCalls != 1 {+ t.Fatalf("removing a provider populated the registry %d times, want 1", store.snapshotCalls)+ }+ if len(store.recordsBySource) != 0 {+ t.Fatalf("a removed provider still supplied records: %v", store.recordsBySource)+ }+}++// The single-source setters keep publishing, so a caller that wires one piece+// at a time still ends with a populated registry.+func TestSingleWiringSettersStillPublish(t *testing.T) {+ store := &testSupplementalResourceStore{}+ monitor := &Monitor{state: models.NewState()}++ monitor.SetResourceStore(store)+ if store.snapshotCalls != 1 {+ t.Fatalf("SetResourceStore populated the registry %d times, want 1", store.snapshotCalls)+ }++ provider := storeWiringProvider("tn-host-1")+ monitor.SetSupplementalRecordsProvider(unifiedresources.SourceTrueNAS, provider)+ if store.snapshotCalls != 2 {+ t.Fatalf("SetSupplementalRecordsProvider populated the registry %d times in all, want 2", store.snapshotCalls)+ }+ if len(store.recordsBySource[unifiedresources.SourceTrueNAS]) != 1 {+ t.Fatalf("the provider's records were not ingested: %v", store.recordsBySource)+ }++ // Unlike the batched wiring step, the single setters are a refresh the+ // caller asked for: the same arguments publish again.+ monitor.SetResourceStore(store)+ monitor.SetSupplementalRecordsProvider(unifiedresources.SourceTrueNAS, provider)+ if store.snapshotCalls != 4 {+ t.Fatalf("repeating the single setters populated the registry %d times in all, want 4", store.snapshotCalls)+ }+}++// This store uses the real in-memory adapter and records every publication,+// rather than only checking the final rows (which could conceal a stale pass).+type wiringPublicationStore struct {+ *unifiedresources.MonitorAdapter+ mu sync.Mutex+ batches [][]string+ before func()+}++func (s *wiringPublicationStore) PopulateSnapshotAndSupplemental(snapshot models.StateSnapshot, records map[unifiedresources.DataSource][]unifiedresources.IngestRecord) {+ if s.before != nil {+ s.before()+ }+ names := []string{}+ for _, batch := range records {+ for _, record := range batch {+ names = append(names, record.SourceID)+ }+ }+ s.mu.Lock()+ s.batches = append(s.batches, names)+ s.mu.Unlock()+ s.MonitorAdapter.PopulateSnapshotAndSupplemental(snapshot, records)+}++func newWiringPublicationStore() *wiringPublicationStore {+ return &wiringPublicationStore{MonitorAdapter: unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil))}+}++type wiringRecordsFunc func(*Monitor, string) []unifiedresources.IngestRecord++func (f wiringRecordsFunc) SupplementalRecords(m *Monitor, org string) []unifiedresources.IngestRecord {+ return f(m, org)+}++func wiringRecords(name string) []unifiedresources.IngestRecord {+ return storeWiringProvider(name).recordsByOrg["default"]+}++// Registration owns a new refresh. A delayed collector is refused before it+// can publish old rows, not repaired by publishing them and repeating a pass.+func TestResourceWiringRefusesCollectedReplacedOrWithdrawnProviders(t *testing.T) {+ for _, withdraw := range []bool{false, true} {+ name := "replace"+ if withdraw {+ name = "withdraw"+ }+ t.Run(name, func(t *testing.T) {+ store := newWiringPublicationStore()+ m := &Monitor{state: models.NewState(), resourceStore: store}+ entered, release, done := make(chan struct{}), make(chan struct{}), make(chan struct{})+ var once sync.Once+ unblock := func() { once.Do(func() { close(release) }) }+ defer unblock()+ m.supplementalProviders = map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider{+ unifiedresources.SourceTrueNAS: wiringRecordsFunc(func(*Monitor, string) []unifiedresources.IngestRecord {+ close(entered)+ <-release+ return wiringRecords("old-host")+ }),+ }+ go func() { defer close(done); m.updateResourceStore(m.currentStateWithScope()) }()+ waitForSignal(t, entered, 5*time.Second, "old collector did not start")+ var replacement MonitorSupplementalRecordsProvider+ if !withdraw {+ replacement = wiringRecordsFunc(func(*Monitor, string) []unifiedresources.IngestRecord { return wiringRecords("new-host") })+ }+ m.SetSupplementalRecordsProvider(unifiedresources.SourceTrueNAS, replacement)+ unblock()+ waitForSignal(t, done, 5*time.Second, "old collector did not return")+ if len(store.batches) != 1 {+ t.Fatalf("published %v, want only the changed binding", store.batches)+ }+ want := 1+ if withdraw {+ want = 0+ }+ if len(store.batches[0]) != want || (!withdraw && store.batches[0][0] != "new-host") {+ t.Fatalf("publications = %v", store.batches)+ }+ for _, resource := range store.GetAll() {+ if resource.Name == "old-host" {+ t.Fatal("withdrawn/replaced host was republished")+ }+ }+ })+ }+}++// A registration that changes during a store write queues its latest publish+// after that write. It cannot finish and then be overwritten by the old pass.+func TestResourceWiringOrdersChangesDuringPublication(t *testing.T) {+ store := newWiringPublicationStore()+ m := &Monitor{state: models.NewState(), resourceStore: store}+ m.supplementalProviders = map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider{+ unifiedresources.SourceTrueNAS: wiringRecordsFunc(func(*Monitor, string) []unifiedresources.IngestRecord { return wiringRecords("old-host") }),+ }+ entered, release, oldDone, newDone := make(chan struct{}), make(chan struct{}), make(chan struct{}), make(chan struct{})+ var once, released sync.Once+ unblock := func() { released.Do(func() { close(release) }) }+ defer unblock()+ store.before = func() { once.Do(func() { close(entered); <-release }) }+ go func() { defer close(oldDone); m.updateResourceStore(m.currentStateWithScope()) }()+ waitForSignal(t, entered, 5*time.Second, "old publication did not start")+ go func() {+ defer close(newDone)+ m.SetSupplementalRecordsProvider(unifiedresources.SourceTrueNAS, wiringRecordsFunc(func(*Monitor, string) []unifiedresources.IngestRecord { return wiringRecords("new-host") }))+ }()+ waitForCondition(t, 5*time.Second, func() bool { return m.resourceWiringGeneration.Load() != 0 }, "new binding was not registered")+ select {+ case <-newDone:+ t.Fatal("new binding completed before the old store write finished")+ default:+ }+ unblock()+ waitForSignal(t, oldDone, 5*time.Second, "old publication did not return")+ waitForSignal(t, newDone, 5*time.Second, "new publication did not return")+ if len(store.batches) != 2 || store.batches[1][0] != "new-host" {+ t.Fatalf("publications = %v, want old then new", store.batches)+ }+ if !m.mockModeFence.registryCurrent() {+ t.Fatal("final registry remained marked stale")+ }+ for _, resource := range store.GetAll() {+ if resource.Name == "old-host" {+ t.Fatal("old row survived the changed binding")+ }+ }+}++func TestResourceWiringProviderCanReenterWithoutPublishingItsOldRecords(t *testing.T) {+ store := newWiringPublicationStore()+ m := &Monitor{state: models.NewState(), resourceStore: store}+ m.supplementalProviders = map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider{+ unifiedresources.SourceTrueNAS: wiringRecordsFunc(func(m *Monitor, _ string) []unifiedresources.IngestRecord {+ m.SetSupplementalRecordsProvider(unifiedresources.SourceTrueNAS, wiringRecordsFunc(func(*Monitor, string) []unifiedresources.IngestRecord { return wiringRecords("new-host") }))+ return wiringRecords("old-host")+ }),+ }+ done := make(chan struct{})+ go func() { defer close(done); m.updateResourceStore(m.currentStateWithScope()) }()+ waitForSignal(t, done, 5*time.Second, "provider re-entry deadlocked")+ if len(store.batches) != 1 || store.batches[0][0] != "new-host" {+ t.Fatalf("publications = %v", store.batches)+ }+}++func TestResourceWiringJoinsAdmittedChecksAndRefusesOldScopes(t *testing.T) {+ m := &Monitor{state: models.NewState()}+ old := m.mockModeFence.begin()+ entered, release, checkDone, wiringDone := make(chan struct{}), make(chan struct{}), make(chan struct{}), make(chan struct{})+ var once sync.Once+ unblock := func() { once.Do(func() { close(release) }) }+ defer unblock()+ go func() { defer close(checkDone); old.run(func() { close(entered); <-release }) }()+ waitForSignal(t, entered, 5*time.Second, "old check did not start")+ go func() {+ defer close(wiringDone)+ m.SetSupplementalRecordsProvider(unifiedresources.SourceTrueNAS, storeWiringProvider("new-host"))+ }()+ waitForCondition(t, 5*time.Second, func() bool { return m.resourceWiringGeneration.Load() != 0 }, "binding did not change")+ select {+ case <-wiringDone:+ t.Fatal("rewiring did not join admitted old checks")+ default:+ }+ unblock()+ waitForSignal(t, checkDone, 5*time.Second, "old check did not finish")+ waitForSignal(t, wiringDone, 5*time.Second, "wiring did not finish")+ if old.run(func() { t.Error("old check was admitted after rewiring") }) {+ t.Fatal("old scope remained usable")+ }+ if !m.mockModeFence.begin().run(func() {}) {+ t.Fatal("new scope refused its check")+ }+}++// Store changes must invalidate a collector too, even with identical providers.+func TestResourceWiringRefusesCollectedRecordsForAReplacedStore(t *testing.T) {+ oldStore, newStore := newWiringPublicationStore(), newWiringPublicationStore()+ m := &Monitor{state: models.NewState(), resourceStore: oldStore}+ entered, release, done := make(chan struct{}), make(chan struct{}), make(chan struct{})+ var held atomic.Bool+ var once sync.Once+ unblock := func() { once.Do(func() { close(release) }) }+ defer unblock()+ p := wiringRecordsFunc(func(*Monitor, string) []unifiedresources.IngestRecord {+ if held.CompareAndSwap(false, true) {+ close(entered)+ <-release+ }+ return wiringRecords("host")+ })+ m.supplementalProviders = map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider{unifiedresources.SourceTrueNAS: p}+ go func() { defer close(done); m.updateResourceStore(m.currentStateWithScope()) }()+ waitForSignal(t, entered, 5*time.Second, "collector did not start")+ m.SetResourceStore(newStore)+ unblock()+ waitForSignal(t, done, 5*time.Second, "collector did not return")+ if len(oldStore.batches) != 0 || len(newStore.batches) != 1 {+ t.Fatalf("old store published %v; new store published %v", oldStore.batches, newStore.batches)+ }+}++type modeBoundWiringProvider struct{ mock bool }++func (p *modeBoundWiringProvider) SupplementalRecordsMockMode() bool { return p.mock }+func (p *modeBoundWiringProvider) SupplementalRecords(*Monitor, string) []unifiedresources.IngestRecord {+ return wiringRecords("bound-host")+}+func (p *modeBoundWiringProvider) SnapshotOwnedSources() []unifiedresources.DataSource {+ return []unifiedresources.DataSource{unifiedresources.SourceTrueNAS}+}++// Exercise both intervals before the post-switch router hook has replaced the+// old pair. Its records and ownership must be absent, not a live/demo mixture.+func TestResourceWiringModeBoundProvidersDoNotCrossTheSwitchHookGap(t *testing.T) {+ previous := mock.IsMockEnabled()+ t.Cleanup(func() { mustSetMockEnabled(t, previous) })+ for _, mode := range []bool{false, true} {+ mustSetMockEnabled(t, mode)+ m := &Monitor{supplementalProviders: map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider{unifiedresources.SourceTrueNAS: &modeBoundWiringProvider{mock: !mode}}}+ if got := m.collectSupplementalRecordsBySource(); len(got) != 0 {+ t.Fatalf("mode %v included opposite-mode records: %v", mode, got)+ }+ if got := m.providerOwnedSnapshotSources(); len(got) != 0 {+ t.Fatalf("mode %v kept opposite-mode snapshot ownership: %v", mode, got)+ }+ m.supplementalProviders[unifiedresources.SourceTrueNAS] = &modeBoundWiringProvider{mock: mode}+ if got := m.collectSupplementalRecordsBySource(); len(got[unifiedresources.SourceTrueNAS]) != 1 {+ t.Fatalf("mode %v lost its matching records: %v", mode, got)+ }+ }+}++func TestResourceWiringDoesNotClearExistingSystemAlerts(t *testing.T) {+ manager := newMockHostAlertTestManager(t)+ if !manager.RaiseSystemAlert(alerts.SystemAlertInput{Type: alerts.BackupEvaluationAlertType, Level: alerts.AlertLevelWarning, Message: "existing condition"}) {+ t.Fatal("control did not create system alert")+ }+ m := &Monitor{state: models.NewState(), alertManager: manager}+ m.SetSupplementalRecordsProvider(unifiedresources.SourceTrueNAS, storeWiringProvider("new-host"))+ active := manager.GetActiveAlerts()+ if len(active) != 1 || active[0].Type != alerts.BackupEvaluationAlertType {+ t.Fatalf("rewiring cleared existing alerts: %v", active)+ }+}++func TestResourceWiringSingleProviderPublishesWithoutLegacyState(t *testing.T) {+ store := newWiringPublicationStore()+ m := &Monitor{resourceStore: store}+ m.SetSupplementalRecordsProvider(unifiedresources.SourceTrueNAS, storeWiringProvider("native-host"))+ if len(store.batches) != 1 || len(store.batches[0]) != 1 || store.batches[0][0] != "native-host" {+ t.Fatalf("source-native backfill without state = %v", store.batches)+ }+}diff --git a/internal/monitoring/truenas_poller.go b/internal/monitoring/truenas_poller.goindex 9c5f784936..c620be45ff 100644--- a/internal/monitoring/truenas_poller.go+++ b/internal/monitoring/truenas_poller.go@@ -993,6 +993,11 @@ func (p *TrueNASPoller) GetCurrentRecords() []unifiedresources.IngestRecord {// SupplementalRecords adapts TrueNAS records to the monitor supplemental// provider contract.+// SupplementalRecordsMockMode binds native records to real infrastructure.+// Mock-mode publications must not read this cached inventory before the+// router's post-switch hook replaces the platform provider pair.+func (*TrueNASPoller) SupplementalRecordsMockMode() bool { return false }+func (p *TrueNASPoller) SupplementalRecords(_ *Monitor, orgID string) []unifiedresources.IngestRecord {return p.GetCurrentRecordsForOrg(orgID)}diff --git a/internal/monitoring/vmware_poller.go b/internal/monitoring/vmware_poller.goindex fc4a7667bd..a2bf7ee945 100644--- a/internal/monitoring/vmware_poller.go+++ b/internal/monitoring/vmware_poller.go@@ -512,6 +512,11 @@ func (p *VMwarePoller) GetCurrentRecords() []unifiedresources.IngestRecord {}// SupplementalRecords implements the monitor supplemental-record provider contract.+// SupplementalRecordsMockMode binds native records to real infrastructure.+// Mock-mode publications must not read this cached inventory before the+// router's post-switch hook replaces the platform provider pair.+func (*VMwarePoller) SupplementalRecordsMockMode() bool { return false }+func (p *VMwarePoller) SupplementalRecords(_ *Monitor, orgID string) []unifiedresources.IngestRecord {return p.GetCurrentRecordsForOrg(orgID)}