Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Keep platform polling independent of organisation metadata
pulse-triage[bot]453eccbba153
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex f09880c2b9..4ef2ff262c 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -17,6 +17,30 @@## Purpose+### Platform connection discovery follows tenant storage identity++TrueNAS and VMware pollers enumerate validated, sorted persistence directory+keys, including `default`, independently of `org.json` metadata. A missing,+malformed or foreign metadata ID cannot prevent a saved connection from+acquiring a provider, redirect its config read, or block authoritative+disable/removal in an unrelated tenant. Configs, credentials, runtime statuses+and discovered records remain keyed by that storage identity and connection+ID. Existing path validation, disabled-connection pruning, encrypted config+loading and poll intervals are unchanged. A directory-list failure still+returns no partial inventory; a connection-config read failure does not gain+an old-credential fallback. Metadata-dependent account and authentication+consumers retain strict organization loading: storage discovery is not+authorization, membership or a substitute for request tenant resolution.++The storage-key, TrueNAS and VMware discovery tests pin malformed/absent IDs,+metadata redirection and unrelated corruption, plus disable/removal without+additional polling. API controls use the actual saved Test, periodic poller+and redacted list response: a probe cannot manufacture runtime evidence, and+ordinary polling supplies transport, attempt/success and observed inventory.+These synthetic source controls reproduce a family matching #2382's second+operator response, not that operator's cause, the original downgrade-resolved+regression, native appliance acceptance or release availability.+### Lightweight detached History readsGuest, node, disk and storage History readers allocate one exact-sized snapshotdiff --git a/internal/api/truenas_handlers_test.go b/internal/api/truenas_handlers_test.goindex f1ba5961ed..bb8a1af321 100644--- a/internal/api/truenas_handlers_test.go+++ b/internal/api/truenas_handlers_test.go@@ -8,6 +8,8 @@ import ("net/http""net/http/httptest""net/url"+ "os"+ "path/filepath""strconv""strings""testing"@@ -1552,3 +1554,78 @@ func trueNASInstanceFromRawURL(t *testing.T, id string, rawURL string, enabled bPollIntervalSecs: 60,}}++// This is an API + real client + periodic poller fixture, not native appliance+// acceptance or attribution of either operator's #2382 installation.+func TestTrueNASHandlers_SavedProbeAndPollingSurviveUnrelatedMetadata(t *testing.T) {+ setTrueNASFeatureForTest(t, true)+ setMockModeForTest(t, false)+ for _, tc := range []struct{ name, metadataDir, body string }{+ {name: "default_empty_id", body: `{}`},+ {name: "default_foreign_id", body: `{"id":"metadata-redirect"}`},+ {name: "unrelated_corrupt_metadata", metadataDir: "orgs/broken-peer", body: `{broken`},+ } {+ t.Run(tc.name, func(t *testing.T) {+ cfg := &config.Config{DataPath: t.TempDir()}+ handler, persistence, _ := newTrueNASHandlersForTest(t, cfg)+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ switch r.URL.Path {+ case "/api/v2.0/system/info":+ _, _ = w.Write([]byte(`{"hostname":"fixture-nas","version":"TrueNAS-SCALE-24.10.2","system_serial":"FIXTURE-NAS"}`))+ case "/api/v2.0/pool", "/api/v2.0/pool/dataset", "/api/v2.0/disk", "/api/v2.0/alert/list":+ _, _ = w.Write([]byte(`[]`))+ default:+ http.NotFound(w, r)+ }+ }))+ t.Cleanup(server.Close)+ connection := trueNASInstanceFromRawURL(t, "fixture-saved-id", server.URL, true)+ if err := persistence.SaveTrueNASConfig([]config.TrueNASInstance{connection}); err != nil {+ t.Fatal(err)+ }+ dir := filepath.Join(cfg.DataPath, tc.metadataDir)+ if err := os.MkdirAll(dir, 0o700); err != nil {+ t.Fatal(err)+ }+ if err := os.WriteFile(filepath.Join(dir, "org.json"), []byte(tc.body), 0o600); err != nil {+ t.Fatal(err)+ }+ poller := monitoring.NewTrueNASPoller(config.NewMultiTenantPersistence(cfg.DataPath), 50*time.Millisecond, nil)+ handler.getPoller = func(context.Context) *monitoring.TrueNASPoller { return poller }+ probe := httptest.NewRecorder()+ handler.HandleTestSavedConnection(probe, httptest.NewRequest(http.MethodPost, "/api/truenas/connections/fixture-saved-id/test", nil))+ if probe.Code != http.StatusOK {+ t.Fatalf("saved probe status = %d: %s", probe.Code, probe.Body.String())+ }+ before := poller.ConnectionSummaries("default", []config.TrueNASInstance{connection})[connection.ID]+ if before.Poll.LastSuccessAt != nil || before.Observed != nil || before.Transport != nil {+ t.Fatal("standalone saved probe manufactured runtime evidence")+ }+ poller.Start(context.Background())+ t.Cleanup(poller.Stop)+ deadline := time.Now().Add(3 * time.Second)+ for time.Now().Before(deadline) {+ summary := poller.ConnectionSummaries("default", []config.TrueNASInstance{connection})[connection.ID]+ if summary.Poll.LastSuccessAt != nil {+ break+ }+ time.Sleep(10 * time.Millisecond)+ }+ listed := httptest.NewRecorder()+ handler.HandleList(listed, httptest.NewRequest(http.MethodGet, "/api/truenas/connections", nil))+ var records []trueNASConnectionResponse+ if listed.Code != http.StatusOK || json.Unmarshal(listed.Body.Bytes(), &records) != nil || len(records) != 1 {+ t.Fatalf("invalid list response: %d %s", listed.Code, listed.Body.String())+ }+ result := records[0]+ if result.Poll == nil || result.Poll.IntervalSeconds != 60 || result.Poll.LastSuccessAt == nil ||+ result.Observed == nil || result.Observed.Host != "fixture-nas" || result.Transport == nil || !result.Transport.Connected {+ t.Fatalf("API did not expose actual saved polling: %s", listed.Body.String())+ }+ if result.APIKey != "********" || strings.Contains(listed.Body.String(), connection.APIKey) {+ t.Fatal("connected runtime summary disclosed stored credentials")+ }+ })+ }+}diff --git a/internal/config/multi_tenant.go b/internal/config/multi_tenant.goindex 1ddc3b3f7d..4bf173b705 100644--- a/internal/config/multi_tenant.go+++ b/internal/config/multi_tenant.go@@ -210,8 +210,12 @@ func (mtp *MultiTenantPersistence) SaveOrganization(org *models.Organization) erreturn nil}-// ListOrganizations returns all known organizations (including the default org).-func (mtp *MultiTenantPersistence) ListOrganizations() ([]*models.Organization, error) {+// ListOrganizationIDs returns sorted persistence keys, including the default+// organization, without reading organization metadata. Background inventory+// readers use these keys so one malformed org.json cannot interrupt unrelated+// monitoring or redirect a connection into another tenant's storage.+// This is storage discovery, not a membership or authorization decision.+func (mtp *MultiTenantPersistence) ListOrganizationIDs() ([]string, error) {orgIDs := map[string]struct{}{"default": {},}@@ -239,9 +243,18 @@ func (mtp *MultiTenantPersistence) ListOrganizations() ([]*models.Organization,sortedIDs = append(sortedIDs, orgID)}sort.Strings(sortedIDs)+ return sortedIDs, nil+}- orgs := make([]*models.Organization, 0, len(sortedIDs))- for _, orgID := range sortedIDs {+// ListOrganizations returns all known organizations (including the default org).+// Metadata-dependent consumers retain strict loading of every organization.+func (mtp *MultiTenantPersistence) ListOrganizations() ([]*models.Organization, error) {+ orgIDs, err := mtp.ListOrganizationIDs()+ if err != nil {+ return nil, err+ }+ orgs := make([]*models.Organization, 0, len(orgIDs))+ for _, orgID := range orgIDs {org, loadErr := mtp.LoadOrganization(orgID)if loadErr != nil {return nil, fmt.Errorf("failed to load organization %s: %w", orgID, loadErr)diff --git a/internal/config/multi_tenant_test.go b/internal/config/multi_tenant_test.goindex 118d479be0..0445cab471 100644--- a/internal/config/multi_tenant_test.go+++ b/internal/config/multi_tenant_test.go@@ -3,6 +3,7 @@ package configimport ("os""path/filepath"+ "reflect""strings""testing")@@ -118,3 +119,73 @@ func TestResolveTenantOrgDirRejectsBlankBaseDir(t *testing.T) {t.Fatal("expected blank base dir to be rejected")}}++func TestMultiTenantPersistence_ListOrganizationIDsUsesStorageKeysOnly(t *testing.T) {+ root := t.TempDir()+ mtp := NewMultiTenantPersistence(root)+ for orgID, body := range map[string]string{+ "default": `{"id":"metadata-redirect"}`,+ "alpha": `{broken`,+ "zeta": `{}`,+ } {+ persistence, err := mtp.GetPersistence(orgID)+ if err != nil {+ t.Fatal(err)+ }+ if err := os.WriteFile(filepath.Join(persistence.GetConfigDir(), "org.json"), []byte(body), 0o600); err != nil {+ t.Fatal(err)+ }+ }+ orgsDir := filepath.Join(root, "orgs")+ for _, invalid := range []string{"bad name", "bad:org"} {+ if err := os.Mkdir(filepath.Join(orgsDir, invalid), 0o700); err != nil {+ t.Fatal(err)+ }+ }+ if err := os.WriteFile(filepath.Join(orgsDir, "stray-file"), []byte("fixture"), 0o600); err != nil {+ t.Fatal(err)+ }+ if err := os.Symlink(t.TempDir(), filepath.Join(orgsDir, "linked-tenant")); err != nil {+ t.Fatal(err)+ }+ got, err := mtp.ListOrganizationIDs()+ if err != nil || !reflect.DeepEqual(got, []string{"alpha", "default", "zeta"}) {+ t.Fatalf("storage IDs = %v, %v", got, err)+ }+ if _, err := os.Stat(filepath.Join(orgsDir, "metadata-redirect")); !os.IsNotExist(err) {+ t.Fatal("enumeration created a tenant from a metadata ID")+ }+ // Metadata-dependent consumers, including account/authentication flows,+ // retain strict decoding: the new storage-key API is not authorization.+ if orgs, err := mtp.ListOrganizations(); err == nil || orgs != nil {+ t.Fatal("metadata list must still fail closed on malformed org.json")+ }+ body, err := os.ReadFile(filepath.Join(orgsDir, "alpha", "org.json"))+ if err != nil || string(body) != "{broken" {+ t.Fatal("enumeration repaired or overwrote metadata")+ }+}++func TestMultiTenantPersistence_ListOrganizationIDsMissingDirectoryAndFailure(t *testing.T) {+ t.Run("missing_directory_has_default_without_writes", func(t *testing.T) {+ root := filepath.Join(t.TempDir(), "absent")+ mtp := NewMultiTenantPersistence(root)+ got, err := mtp.ListOrganizationIDs()+ if err != nil || !reflect.DeepEqual(got, []string{"default"}) {+ t.Fatalf("missing directory IDs = %v, %v", got, err)+ }+ if _, err := os.Stat(root); !os.IsNotExist(err) {+ t.Fatal("ID enumeration created persistence or metadata")+ }+ })+ t.Run("directory_failure_returns_no_partial_inventory", func(t *testing.T) {+ root := t.TempDir()+ if err := os.WriteFile(filepath.Join(root, "orgs"), []byte("fixture"), 0o600); err != nil {+ t.Fatal(err)+ }+ got, err := NewMultiTenantPersistence(root).ListOrganizationIDs()+ if err == nil || got != nil {+ t.Fatalf("broken organization directory returned partial inventory: %v, %v", got, err)+ }+ })+}diff --git a/internal/monitoring/truenas_poller.go b/internal/monitoring/truenas_poller.goindex debcdc8b7a..e8e627fe71 100644--- a/internal/monitoring/truenas_poller.go+++ b/internal/monitoring/truenas_poller.go@@ -185,7 +185,7 @@ func (p *TrueNASPoller) syncConnections() {return}- orgs, err := p.multiTenant.ListOrganizations()+ orgIDs, err := p.multiTenant.ListOrganizationIDs()if err != nil {log.Warn().Str("component", "truenas_poller").@@ -196,17 +196,10 @@ func (p *TrueNASPoller) syncConnections() {}// orgID -> connID -> instance- configured := make(map[string]map[string]config.TrueNASInstance, len(orgs))- active := make(map[string]map[string]config.TrueNASInstance, len(orgs))+ configured := make(map[string]map[string]config.TrueNASInstance, len(orgIDs))+ active := make(map[string]map[string]config.TrueNASInstance, len(orgIDs))- for _, org := range orgs {- if org == nil {- continue- }- orgID := strings.TrimSpace(org.ID)- if orgID == "" {- continue- }+ for _, orgID := range orgIDs {persistence, err := p.multiTenant.GetPersistence(orgID)if err != nil || persistence == nil {log.Warn().diff --git a/internal/monitoring/truenas_poller_test.go b/internal/monitoring/truenas_poller_test.goindex 290814828e..505ddcc393 100644--- a/internal/monitoring/truenas_poller_test.go+++ b/internal/monitoring/truenas_poller_test.go@@ -11,6 +11,7 @@ import ("net/http/httptest""net/url""os"+ "path/filepath""strconv""strings""sync/atomic"@@ -2720,3 +2721,229 @@ func TestTrueNASPartialReportingPipeline(t *testing.T) {t.Fatalf("sufficiently covered local History lost Thermals: %+v", points)}}++func setDiscoveryOrgMetadata(t *testing.T, persistence *config.ConfigPersistence, body string) {+ t.Helper()+ if err := os.WriteFile(filepath.Join(persistence.GetConfigDir(), "org.json"), []byte(body), 0o600); err != nil {+ t.Fatal(err)+ }+}++func discoveryTenant(t *testing.T, mtp *config.MultiTenantPersistence, orgID string) *config.ConfigPersistence {+ t.Helper()+ persistence, err := mtp.GetPersistence(orgID)+ if err != nil {+ t.Fatal(err)+ }+ return persistence+}++func TestTrueNASPollerStorageIdentityDiscovery(t *testing.T) {+ for _, tc := range []struct {+ name, orgID, metadata, peerMetadata string+ }{+ {name: "normal_missing_metadata", orgID: "default"},+ {name: "default_malformed_metadata", orgID: "default", metadata: "{broken"},+ {name: "default_missing_metadata_id", orgID: "default", metadata: `{}`},+ {name: "default_metadata_redirect", orgID: "default", metadata: `{"id":"metadata-redirect"}`},+ {name: "unrelated_malformed_metadata", orgID: "default", peerMetadata: "{broken"},+ {name: "tenant_malformed_metadata", orgID: "tenant-a", metadata: "{broken"},+ {name: "tenant_metadata_collision", orgID: "tenant-a", metadata: `{"id":"tenant-b"}`, peerMetadata: `{"id":"tenant-b"}`},+ } {+ t.Run(tc.name, func(t *testing.T) {+ server := newTrueNASMockServer(t, "storage-owner-a")+ t.Cleanup(server.Close)+ mtp, _ := newTestTenantPersistence(t)+ persistence := discoveryTenant(t, mtp, tc.orgID)+ connection := trueNASInstanceForServer(t, "same-local-id", server.URL(), true)+ connection.Username = "fixture-owner"+ if err := persistence.SaveTrueNASConfig([]config.TrueNASInstance{connection}); err != nil {+ t.Fatal(err)+ }+ if tc.metadata != "" {+ setDiscoveryOrgMetadata(t, persistence, tc.metadata)+ }+ if tc.peerMetadata != "" {+ peer := discoveryTenant(t, mtp, "tenant-b")+ setDiscoveryOrgMetadata(t, peer, tc.peerMetadata)+ }++ // An actual standalone probe reads system.info even when the+ // poller's organization discovery cannot reach this saved config.+ probe, err := truenas.NewClient(truenas.ClientConfig{+ Host: connection.Host, Port: connection.Port,+ UseHTTPS: connection.UseHTTPS, APIKey: connection.APIKey, Username: connection.Username,+ })+ if err != nil {+ t.Fatal(err)+ }+ ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)+ defer cancel()+ defer probe.Close()+ if err := probe.TestConnection(ctx); err != nil {+ t.Fatalf("standalone probe failed: %v", err)+ }++ poller := NewTrueNASPoller(mtp, 0, nil)+ t.Cleanup(func() {+ for _, byConnection := range poller.providersByOrg {+ for _, provider := range byConnection {+ provider.Close()+ }+ }+ })+ poller.syncConnections()+ if poller.providersByOrg[tc.orgID][connection.ID] == nil {+ projection, _ := json.Marshal(poller.ConnectionSummaries(tc.orgID, []config.TrueNASInstance{connection})[connection.ID])+ t.Fatalf("standalone probe succeeded but saved provider is absent: %s", projection)+ }+ poller.pollAll(ctx)+ summary := poller.ConnectionSummaries(tc.orgID, []config.TrueNASInstance{connection})[connection.ID]+ if summary.Poll == nil || summary.Poll.LastSuccessAt == nil || summary.Poll.LastAttemptAt == nil ||+ summary.Observed == nil || summary.Observed.Host != "storage-owner-a" || summary.Transport == nil ||+ !summary.Transport.Connected || !hasTrueNASHostForOrg(poller, tc.orgID, "storage-owner-a") {+ t.Fatalf("ordinary polling did not reach saved connection: %+v", summary)+ }+ for _, other := range []string{"default", "tenant-b", "metadata-redirect"} {+ if other != tc.orgID && len(poller.GetCurrentRecordsForOrg(other)) != 0 {+ t.Fatalf("resources escaped storage owner into %s", other)+ }+ }+ if _, err := os.Stat(filepath.Join(mtp.BaseDataDir(), "orgs", "metadata-redirect")); !os.IsNotExist(err) {+ t.Fatal("metadata created an unrelated tenant directory")+ }+ })+ }+}++func TestTrueNASPollerStorageDiscoveryAppliesRevocationWithBrokenPeerMetadata(t *testing.T) {+ for _, action := range []string{"disable", "remove"} {+ t.Run(action, func(t *testing.T) {+ server := newTrueNASMockServer(t, "revocation-owner")+ t.Cleanup(server.Close)+ mtp, persistence := newTestTenantPersistence(t)+ connection := trueNASInstanceForServer(t, "revocation-id", server.URL(), true)+ if err := persistence.SaveTrueNASConfig([]config.TrueNASInstance{connection}); err != nil {+ t.Fatal(err)+ }+ poller := NewTrueNASPoller(mtp, 0, nil)+ t.Cleanup(func() {+ for _, byConnection := range poller.providersByOrg {+ for _, provider := range byConnection {+ provider.Close()+ }+ }+ })+ poller.syncConnections()+ poller.pollAll(context.Background())+ if !hasTrueNASHostForOrg(poller, "default", "revocation-owner") {+ t.Fatal("initial healthy fixture did not poll")+ }+ before := server.RequestCount()+ setDiscoveryOrgMetadata(t, discoveryTenant(t, mtp, "broken-peer"), "{broken")+ next := []config.TrueNASInstance{}+ if action == "disable" {+ connection.Enabled = false+ next = append(next, connection)+ }+ if err := persistence.SaveTrueNASConfig(next); err != nil {+ t.Fatal(err)+ }+ poller.syncConnections()+ if poller.providersByOrg["default"][connection.ID] != nil {+ t.Fatal("unrelated org metadata blocked authoritative connection revocation")+ }+ poller.pollAll(context.Background())+ if server.RequestCount() != before || len(poller.GetCurrentRecordsForOrg("default")) != 0 {+ t.Fatal("revoked connection still polled or contributed current records")+ }+ if action == "remove" && poller.statusByOrg["default"][connection.ID] != nil {+ t.Fatal("removed connection retained runtime state")+ }+ })+ }+}++func TestTrueNASPollerStorageIdentitySeparatesTenantsAndTracksRemoval(t *testing.T) {+ mtp, _ := newTestTenantPersistence(t)+ servers := map[string]*trueNASMockServer{}+ for _, orgID := range []string{"tenant-a", "tenant-b"} {+ server := newTrueNASMockServer(t, orgID+"-nas")+ t.Cleanup(server.Close)+ servers[orgID] = server+ connection := trueNASInstanceForServer(t, "same-local-id", server.URL(), true)+ persistence := discoveryTenant(t, mtp, orgID)+ if err := persistence.SaveTrueNASConfig([]config.TrueNASInstance{connection}); err != nil {+ t.Fatal(err)+ }+ setDiscoveryOrgMetadata(t, persistence, `{"id":"tenant-b"}`)+ }+ poller := NewTrueNASPoller(mtp, 0, nil)+ t.Cleanup(func() {+ for _, byConnection := range poller.providersByOrg {+ for _, provider := range byConnection {+ provider.Close()+ }+ }+ })+ poller.syncConnections()+ poller.pollAll(context.Background())+ for _, orgID := range []string{"tenant-a", "tenant-b"} {+ if !hasTrueNASHostForOrg(poller, orgID, orgID+"-nas") {+ t.Fatalf("missing own tenant's resources: %s", orgID)+ }+ for _, other := range []string{"tenant-a", "tenant-b"} {+ if other != orgID && hasTrueNASHostForOrg(poller, orgID, other+"-nas") {+ t.Fatal("same local connection ID merged different tenant inventories")+ }+ }+ }+ retained := poller.providersByOrg["tenant-b"]["same-local-id"]+ removedRequests := servers["tenant-a"].RequestCount()+ if err := mtp.DeleteOrganization("tenant-a"); err != nil {+ t.Fatal(err)+ }+ poller.syncConnections()+ poller.pollAll(context.Background())+ if len(poller.GetCurrentRecordsForOrg("tenant-a")) != 0 || len(poller.providersByOrg["tenant-a"]) != 0 ||+ len(poller.statusByOrg["tenant-a"]) != 0 || servers["tenant-a"].RequestCount() != removedRequests {+ t.Fatal("removed tenant retained runtime inventory or kept polling")+ }+ if poller.providersByOrg["tenant-b"]["same-local-id"] != retained ||+ !hasTrueNASHostForOrg(poller, "tenant-b", "tenant-b-nas") {+ t.Fatal("removed tenant disrupted a different storage owner's provider")+ }+}++func TestTrueNASPollerStorageDiscoveryDoesNotReuseUnreadableConnectionConfig(t *testing.T) {+ server := newTrueNASMockServer(t, "config-read-owner")+ t.Cleanup(server.Close)+ mtp, persistence := newTestTenantPersistence(t)+ connection := trueNASInstanceForServer(t, "config-read-id", server.URL(), true)+ if err := persistence.SaveTrueNASConfig([]config.TrueNASInstance{connection}); err != nil {+ t.Fatal(err)+ }+ poller := NewTrueNASPoller(mtp, 0, nil)+ t.Cleanup(func() {+ for _, byConnection := range poller.providersByOrg {+ for _, provider := range byConnection {+ provider.Close()+ }+ }+ })+ poller.syncConnections()+ poller.pollAll(context.Background())+ if !hasTrueNASHostForOrg(poller, "default", "config-read-owner") {+ t.Fatal("initial fixture did not poll")+ }+ before := server.RequestCount()+ if err := os.WriteFile(filepath.Join(persistence.GetConfigDir(), "truenas.enc"), []byte("{broken"), 0o600); err != nil {+ t.Fatal(err)+ }+ poller.syncConnections()+ poller.pollAll(context.Background())+ if poller.providersByOrg["default"][connection.ID] != nil || server.RequestCount() != before ||+ len(poller.GetCurrentRecordsForOrg("default")) != 0 {+ t.Fatal("unreadable authoritative config reused old credentials or records")+ }+}diff --git a/internal/monitoring/vmware_poller.go b/internal/monitoring/vmware_poller.goindex 572a40aa13..fc4a7667bd 100644--- a/internal/monitoring/vmware_poller.go+++ b/internal/monitoring/vmware_poller.go@@ -193,7 +193,7 @@ func (p *VMwarePoller) syncConnections() {return}- orgs, err := p.multiTenant.ListOrganizations()+ orgIDs, err := p.multiTenant.ListOrganizationIDs()if err != nil {log.Warn().Str("component", "vmware_poller").@@ -203,15 +203,8 @@ func (p *VMwarePoller) syncConnections() {return}- active := make(map[string]map[string]config.VMwareVCenterInstance, len(orgs))- for _, org := range orgs {- if org == nil {- continue- }- orgID := strings.TrimSpace(org.ID)- if orgID == "" {- continue- }+ active := make(map[string]map[string]config.VMwareVCenterInstance, len(orgIDs))+ for _, orgID := range orgIDs {persistence, err := p.multiTenant.GetPersistence(orgID)if err != nil || persistence == nil {log.Warn().diff --git a/internal/monitoring/vmware_poller_test.go b/internal/monitoring/vmware_poller_test.goindex 448562421d..92a3a649a4 100644--- a/internal/monitoring/vmware_poller_test.go+++ b/internal/monitoring/vmware_poller_test.go@@ -3,6 +3,8 @@ package monitoringimport ("context""errors"+ "os"+ "path/filepath""strings""testing""time"@@ -481,3 +483,104 @@ func captureVMwarePollerLogs(t *testing.T) *monitoringLogCapture {t.Helper()return newMonitoringLogCapture(t)}++func TestVMwarePollerStorageIdentityDiscovery(t *testing.T) {+ for _, tc := range []struct{ name, metadata, peerMetadata string }{+ {name: "normal_missing_metadata"},+ {name: "malformed_metadata", metadata: "{broken"},+ {name: "missing_metadata_id", metadata: `{}`},+ {name: "metadata_redirect", metadata: `{"id":"metadata-redirect"}`},+ {name: "unrelated_malformed_metadata", peerMetadata: "{broken"},+ } {+ t.Run(tc.name, func(t *testing.T) {+ mtp, persistence := newTestTenantPersistence(t)+ connection := config.NewVMwareVCenterInstance()+ connection.ID = "vc-storage-id"+ connection.Host = "fixture-vcenter"+ connection.Username = "fixture-owner"+ connection.Password = "fixture-password"+ if err := persistence.SaveVMwareConfig([]config.VMwareVCenterInstance{connection}); err != nil {+ t.Fatal(err)+ }+ if tc.metadata != "" {+ setDiscoveryOrgMetadata(t, persistence, tc.metadata)+ }+ if tc.peerMetadata != "" {+ setDiscoveryOrgMetadata(t, discoveryTenant(t, mtp, "broken-peer"), tc.peerMetadata)+ }+ poller := NewVMwarePoller(mtp, 0)+ poller.newProvider = func(instance config.VMwareVCenterInstance) (vmwarePollerProvider, error) {+ return vmware.NewProvider(vmware.InventorySnapshot{+ ConnectionID: instance.ID, CollectedAt: time.Now().UTC(),+ Hosts: []vmware.InventoryHost{{Host: "host-1", Name: "storage-owned-host"}},+ }), nil+ }+ t.Cleanup(func() {+ for _, byConnection := range poller.providersByOrg {+ for _, provider := range byConnection {+ provider.Close()+ }+ }+ })+ poller.syncConnections()+ poller.pollAll(context.Background())+ summary := poller.ConnectionSummaries("default", []config.VMwareVCenterInstance{connection})[connection.ID]+ if summary.Poll == nil || summary.Poll.LastSuccessAt == nil || summary.Observed == nil ||+ summary.Observed.Hosts != 1 || len(poller.GetCurrentRecordsForOrg("default")) != 1 {+ t.Fatalf("saved VMware config was not discovered independently of metadata: %+v", summary)+ }+ if _, err := os.Stat(filepath.Join(mtp.BaseDataDir(), "orgs", "metadata-redirect")); !os.IsNotExist(err) {+ t.Fatal("metadata created an unrelated tenant directory")+ }+ })+ }+}++func TestVMwarePollerStorageDiscoveryAppliesRevocationWithBrokenPeerMetadata(t *testing.T) {+ for _, action := range []string{"disable", "remove", "unreadable_config"} {+ t.Run(action, func(t *testing.T) {+ mtp, persistence := newTestTenantPersistence(t)+ connection := config.NewVMwareVCenterInstance()+ connection.ID = "vc-storage-id"+ connection.Host = "fixture-vcenter"+ connection.Username = "fixture-owner"+ connection.Password = "fixture-password"+ if err := persistence.SaveVMwareConfig([]config.VMwareVCenterInstance{connection}); err != nil {+ t.Fatal(err)+ }+ poller := NewVMwarePoller(mtp, 0)+ poller.newProvider = func(instance config.VMwareVCenterInstance) (vmwarePollerProvider, error) {+ return vmware.NewProvider(vmware.InventorySnapshot{+ ConnectionID: instance.ID, CollectedAt: time.Now().UTC(),+ Hosts: []vmware.InventoryHost{{Host: "host-1", Name: "storage-owned-host"}},+ }), nil+ }+ poller.syncConnections()+ poller.pollAll(context.Background())+ if len(poller.GetCurrentRecordsForOrg("default")) != 1 {+ t.Fatal("initial fixture did not poll")+ }+ setDiscoveryOrgMetadata(t, discoveryTenant(t, mtp, "broken-peer"), "{broken")+ switch action {+ case "disable":+ connection.Enabled = false+ if err := persistence.SaveVMwareConfig([]config.VMwareVCenterInstance{connection}); err != nil {+ t.Fatal(err)+ }+ case "remove":+ if err := persistence.SaveVMwareConfig(nil); err != nil {+ t.Fatal(err)+ }+ case "unreadable_config":+ if err := os.WriteFile(filepath.Join(persistence.GetConfigDir(), "vmware.enc"), []byte("{broken"), 0o600); err != nil {+ t.Fatal(err)+ }+ }+ poller.syncConnections()+ poller.pollAll(context.Background())+ if poller.providersByOrg["default"][connection.ID] != nil || len(poller.GetCurrentRecordsForOrg("default")) != 0 {+ t.Fatal("unrelated metadata blocked fail-closed config reconciliation")+ }+ })+ }+}