Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Keep platform polling independent of organisation metadata

pulse-triage[bot]453eccbba153
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index f09880c2b9..4ef2ff262c 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -17,6 +17,30 @@
## Purpose
+### Platform connection discovery follows tenant storage identity
+
+TrueNAS and VMware pollers enumerate validated, sorted persistence directory
+keys, including `default`, independently of `org.json` metadata. A missing,
+malformed or foreign metadata ID cannot prevent a saved connection from
+acquiring a provider, redirect its config read, or block authoritative
+disable/removal in an unrelated tenant. Configs, credentials, runtime statuses
+and discovered records remain keyed by that storage identity and connection
+ID. Existing path validation, disabled-connection pruning, encrypted config
+loading and poll intervals are unchanged. A directory-list failure still
+returns no partial inventory; a connection-config read failure does not gain
+an old-credential fallback. Metadata-dependent account and authentication
+consumers retain strict organization loading: storage discovery is not
+authorization, membership or a substitute for request tenant resolution.
+
+The storage-key, TrueNAS and VMware discovery tests pin malformed/absent IDs,
+metadata redirection and unrelated corruption, plus disable/removal without
+additional polling. API controls use the actual saved Test, periodic poller
+and redacted list response: a probe cannot manufacture runtime evidence, and
+ordinary polling supplies transport, attempt/success and observed inventory.
+These synthetic source controls reproduce a family matching #2382's second
+operator response, not that operator's cause, the original downgrade-resolved
+regression, native appliance acceptance or release availability.
+
### Lightweight detached History reads
Guest, node, disk and storage History readers allocate one exact-sized snapshot
diff --git a/internal/api/truenas_handlers_test.go b/internal/api/truenas_handlers_test.go
index f1ba5961ed..bb8a1af321 100644
--- a/internal/api/truenas_handlers_test.go
+++ b/internal/api/truenas_handlers_test.go
@@ -8,6 +8,8 @@ import (
"net/http"
"net/http/httptest"
"net/url"
+ "os"
+ "path/filepath"
"strconv"
"strings"
"testing"
@@ -1552,3 +1554,78 @@ func trueNASInstanceFromRawURL(t *testing.T, id string, rawURL string, enabled b
PollIntervalSecs: 60,
}
}
+
+// This is an API + real client + periodic poller fixture, not native appliance
+// acceptance or attribution of either operator's #2382 installation.
+func TestTrueNASHandlers_SavedProbeAndPollingSurviveUnrelatedMetadata(t *testing.T) {
+ setTrueNASFeatureForTest(t, true)
+ setMockModeForTest(t, false)
+ for _, tc := range []struct{ name, metadataDir, body string }{
+ {name: "default_empty_id", body: `{}`},
+ {name: "default_foreign_id", body: `{"id":"metadata-redirect"}`},
+ {name: "unrelated_corrupt_metadata", metadataDir: "orgs/broken-peer", body: `{broken`},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ cfg := &config.Config{DataPath: t.TempDir()}
+ handler, persistence, _ := newTrueNASHandlersForTest(t, cfg)
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ switch r.URL.Path {
+ case "/api/v2.0/system/info":
+ _, _ = w.Write([]byte(`{"hostname":"fixture-nas","version":"TrueNAS-SCALE-24.10.2","system_serial":"FIXTURE-NAS"}`))
+ case "/api/v2.0/pool", "/api/v2.0/pool/dataset", "/api/v2.0/disk", "/api/v2.0/alert/list":
+ _, _ = w.Write([]byte(`[]`))
+ default:
+ http.NotFound(w, r)
+ }
+ }))
+ t.Cleanup(server.Close)
+ connection := trueNASInstanceFromRawURL(t, "fixture-saved-id", server.URL, true)
+ if err := persistence.SaveTrueNASConfig([]config.TrueNASInstance{connection}); err != nil {
+ t.Fatal(err)
+ }
+ dir := filepath.Join(cfg.DataPath, tc.metadataDir)
+ if err := os.MkdirAll(dir, 0o700); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(filepath.Join(dir, "org.json"), []byte(tc.body), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ poller := monitoring.NewTrueNASPoller(config.NewMultiTenantPersistence(cfg.DataPath), 50*time.Millisecond, nil)
+ handler.getPoller = func(context.Context) *monitoring.TrueNASPoller { return poller }
+ probe := httptest.NewRecorder()
+ handler.HandleTestSavedConnection(probe, httptest.NewRequest(http.MethodPost, "/api/truenas/connections/fixture-saved-id/test", nil))
+ if probe.Code != http.StatusOK {
+ t.Fatalf("saved probe status = %d: %s", probe.Code, probe.Body.String())
+ }
+ before := poller.ConnectionSummaries("default", []config.TrueNASInstance{connection})[connection.ID]
+ if before.Poll.LastSuccessAt != nil || before.Observed != nil || before.Transport != nil {
+ t.Fatal("standalone saved probe manufactured runtime evidence")
+ }
+ poller.Start(context.Background())
+ t.Cleanup(poller.Stop)
+ deadline := time.Now().Add(3 * time.Second)
+ for time.Now().Before(deadline) {
+ summary := poller.ConnectionSummaries("default", []config.TrueNASInstance{connection})[connection.ID]
+ if summary.Poll.LastSuccessAt != nil {
+ break
+ }
+ time.Sleep(10 * time.Millisecond)
+ }
+ listed := httptest.NewRecorder()
+ handler.HandleList(listed, httptest.NewRequest(http.MethodGet, "/api/truenas/connections", nil))
+ var records []trueNASConnectionResponse
+ if listed.Code != http.StatusOK || json.Unmarshal(listed.Body.Bytes(), &records) != nil || len(records) != 1 {
+ t.Fatalf("invalid list response: %d %s", listed.Code, listed.Body.String())
+ }
+ result := records[0]
+ if result.Poll == nil || result.Poll.IntervalSeconds != 60 || result.Poll.LastSuccessAt == nil ||
+ result.Observed == nil || result.Observed.Host != "fixture-nas" || result.Transport == nil || !result.Transport.Connected {
+ t.Fatalf("API did not expose actual saved polling: %s", listed.Body.String())
+ }
+ if result.APIKey != "********" || strings.Contains(listed.Body.String(), connection.APIKey) {
+ t.Fatal("connected runtime summary disclosed stored credentials")
+ }
+ })
+ }
+}
diff --git a/internal/config/multi_tenant.go b/internal/config/multi_tenant.go
index 1ddc3b3f7d..4bf173b705 100644
--- a/internal/config/multi_tenant.go
+++ b/internal/config/multi_tenant.go
@@ -210,8 +210,12 @@ func (mtp *MultiTenantPersistence) SaveOrganization(org *models.Organization) er
return nil
}
-// ListOrganizations returns all known organizations (including the default org).
-func (mtp *MultiTenantPersistence) ListOrganizations() ([]*models.Organization, error) {
+// ListOrganizationIDs returns sorted persistence keys, including the default
+// organization, without reading organization metadata. Background inventory
+// readers use these keys so one malformed org.json cannot interrupt unrelated
+// monitoring or redirect a connection into another tenant's storage.
+// This is storage discovery, not a membership or authorization decision.
+func (mtp *MultiTenantPersistence) ListOrganizationIDs() ([]string, error) {
orgIDs := map[string]struct{}{
"default": {},
}
@@ -239,9 +243,18 @@ func (mtp *MultiTenantPersistence) ListOrganizations() ([]*models.Organization,
sortedIDs = append(sortedIDs, orgID)
}
sort.Strings(sortedIDs)
+ return sortedIDs, nil
+}
- orgs := make([]*models.Organization, 0, len(sortedIDs))
- for _, orgID := range sortedIDs {
+// ListOrganizations returns all known organizations (including the default org).
+// Metadata-dependent consumers retain strict loading of every organization.
+func (mtp *MultiTenantPersistence) ListOrganizations() ([]*models.Organization, error) {
+ orgIDs, err := mtp.ListOrganizationIDs()
+ if err != nil {
+ return nil, err
+ }
+ orgs := make([]*models.Organization, 0, len(orgIDs))
+ for _, orgID := range orgIDs {
org, loadErr := mtp.LoadOrganization(orgID)
if loadErr != nil {
return nil, fmt.Errorf("failed to load organization %s: %w", orgID, loadErr)
diff --git a/internal/config/multi_tenant_test.go b/internal/config/multi_tenant_test.go
index 118d479be0..0445cab471 100644
--- a/internal/config/multi_tenant_test.go
+++ b/internal/config/multi_tenant_test.go
@@ -3,6 +3,7 @@ package config
import (
"os"
"path/filepath"
+ "reflect"
"strings"
"testing"
)
@@ -118,3 +119,73 @@ func TestResolveTenantOrgDirRejectsBlankBaseDir(t *testing.T) {
t.Fatal("expected blank base dir to be rejected")
}
}
+
+func TestMultiTenantPersistence_ListOrganizationIDsUsesStorageKeysOnly(t *testing.T) {
+ root := t.TempDir()
+ mtp := NewMultiTenantPersistence(root)
+ for orgID, body := range map[string]string{
+ "default": `{"id":"metadata-redirect"}`,
+ "alpha": `{broken`,
+ "zeta": `{}`,
+ } {
+ persistence, err := mtp.GetPersistence(orgID)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(filepath.Join(persistence.GetConfigDir(), "org.json"), []byte(body), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ }
+ orgsDir := filepath.Join(root, "orgs")
+ for _, invalid := range []string{"bad name", "bad:org"} {
+ if err := os.Mkdir(filepath.Join(orgsDir, invalid), 0o700); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if err := os.WriteFile(filepath.Join(orgsDir, "stray-file"), []byte("fixture"), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Symlink(t.TempDir(), filepath.Join(orgsDir, "linked-tenant")); err != nil {
+ t.Fatal(err)
+ }
+ got, err := mtp.ListOrganizationIDs()
+ if err != nil || !reflect.DeepEqual(got, []string{"alpha", "default", "zeta"}) {
+ t.Fatalf("storage IDs = %v, %v", got, err)
+ }
+ if _, err := os.Stat(filepath.Join(orgsDir, "metadata-redirect")); !os.IsNotExist(err) {
+ t.Fatal("enumeration created a tenant from a metadata ID")
+ }
+ // Metadata-dependent consumers, including account/authentication flows,
+ // retain strict decoding: the new storage-key API is not authorization.
+ if orgs, err := mtp.ListOrganizations(); err == nil || orgs != nil {
+ t.Fatal("metadata list must still fail closed on malformed org.json")
+ }
+ body, err := os.ReadFile(filepath.Join(orgsDir, "alpha", "org.json"))
+ if err != nil || string(body) != "{broken" {
+ t.Fatal("enumeration repaired or overwrote metadata")
+ }
+}
+
+func TestMultiTenantPersistence_ListOrganizationIDsMissingDirectoryAndFailure(t *testing.T) {
+ t.Run("missing_directory_has_default_without_writes", func(t *testing.T) {
+ root := filepath.Join(t.TempDir(), "absent")
+ mtp := NewMultiTenantPersistence(root)
+ got, err := mtp.ListOrganizationIDs()
+ if err != nil || !reflect.DeepEqual(got, []string{"default"}) {
+ t.Fatalf("missing directory IDs = %v, %v", got, err)
+ }
+ if _, err := os.Stat(root); !os.IsNotExist(err) {
+ t.Fatal("ID enumeration created persistence or metadata")
+ }
+ })
+ t.Run("directory_failure_returns_no_partial_inventory", func(t *testing.T) {
+ root := t.TempDir()
+ if err := os.WriteFile(filepath.Join(root, "orgs"), []byte("fixture"), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ got, err := NewMultiTenantPersistence(root).ListOrganizationIDs()
+ if err == nil || got != nil {
+ t.Fatalf("broken organization directory returned partial inventory: %v, %v", got, err)
+ }
+ })
+}
diff --git a/internal/monitoring/truenas_poller.go b/internal/monitoring/truenas_poller.go
index debcdc8b7a..e8e627fe71 100644
--- a/internal/monitoring/truenas_poller.go
+++ b/internal/monitoring/truenas_poller.go
@@ -185,7 +185,7 @@ func (p *TrueNASPoller) syncConnections() {
return
}
- orgs, err := p.multiTenant.ListOrganizations()
+ orgIDs, err := p.multiTenant.ListOrganizationIDs()
if err != nil {
log.Warn().
Str("component", "truenas_poller").
@@ -196,17 +196,10 @@ func (p *TrueNASPoller) syncConnections() {
}
// orgID -> connID -> instance
- configured := make(map[string]map[string]config.TrueNASInstance, len(orgs))
- active := make(map[string]map[string]config.TrueNASInstance, len(orgs))
+ configured := make(map[string]map[string]config.TrueNASInstance, len(orgIDs))
+ active := make(map[string]map[string]config.TrueNASInstance, len(orgIDs))
- for _, org := range orgs {
- if org == nil {
- continue
- }
- orgID := strings.TrimSpace(org.ID)
- if orgID == "" {
- continue
- }
+ for _, orgID := range orgIDs {
persistence, err := p.multiTenant.GetPersistence(orgID)
if err != nil || persistence == nil {
log.Warn().
diff --git a/internal/monitoring/truenas_poller_test.go b/internal/monitoring/truenas_poller_test.go
index 290814828e..505ddcc393 100644
--- a/internal/monitoring/truenas_poller_test.go
+++ b/internal/monitoring/truenas_poller_test.go
@@ -11,6 +11,7 @@ import (
"net/http/httptest"
"net/url"
"os"
+ "path/filepath"
"strconv"
"strings"
"sync/atomic"
@@ -2720,3 +2721,229 @@ func TestTrueNASPartialReportingPipeline(t *testing.T) {
t.Fatalf("sufficiently covered local History lost Thermals: %+v", points)
}
}
+
+func setDiscoveryOrgMetadata(t *testing.T, persistence *config.ConfigPersistence, body string) {
+ t.Helper()
+ if err := os.WriteFile(filepath.Join(persistence.GetConfigDir(), "org.json"), []byte(body), 0o600); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func discoveryTenant(t *testing.T, mtp *config.MultiTenantPersistence, orgID string) *config.ConfigPersistence {
+ t.Helper()
+ persistence, err := mtp.GetPersistence(orgID)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return persistence
+}
+
+func TestTrueNASPollerStorageIdentityDiscovery(t *testing.T) {
+ for _, tc := range []struct {
+ name, orgID, metadata, peerMetadata string
+ }{
+ {name: "normal_missing_metadata", orgID: "default"},
+ {name: "default_malformed_metadata", orgID: "default", metadata: "{broken"},
+ {name: "default_missing_metadata_id", orgID: "default", metadata: `{}`},
+ {name: "default_metadata_redirect", orgID: "default", metadata: `{"id":"metadata-redirect"}`},
+ {name: "unrelated_malformed_metadata", orgID: "default", peerMetadata: "{broken"},
+ {name: "tenant_malformed_metadata", orgID: "tenant-a", metadata: "{broken"},
+ {name: "tenant_metadata_collision", orgID: "tenant-a", metadata: `{"id":"tenant-b"}`, peerMetadata: `{"id":"tenant-b"}`},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ server := newTrueNASMockServer(t, "storage-owner-a")
+ t.Cleanup(server.Close)
+ mtp, _ := newTestTenantPersistence(t)
+ persistence := discoveryTenant(t, mtp, tc.orgID)
+ connection := trueNASInstanceForServer(t, "same-local-id", server.URL(), true)
+ connection.Username = "fixture-owner"
+ if err := persistence.SaveTrueNASConfig([]config.TrueNASInstance{connection}); err != nil {
+ t.Fatal(err)
+ }
+ if tc.metadata != "" {
+ setDiscoveryOrgMetadata(t, persistence, tc.metadata)
+ }
+ if tc.peerMetadata != "" {
+ peer := discoveryTenant(t, mtp, "tenant-b")
+ setDiscoveryOrgMetadata(t, peer, tc.peerMetadata)
+ }
+
+ // An actual standalone probe reads system.info even when the
+ // poller's organization discovery cannot reach this saved config.
+ probe, err := truenas.NewClient(truenas.ClientConfig{
+ Host: connection.Host, Port: connection.Port,
+ UseHTTPS: connection.UseHTTPS, APIKey: connection.APIKey, Username: connection.Username,
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+ defer cancel()
+ defer probe.Close()
+ if err := probe.TestConnection(ctx); err != nil {
+ t.Fatalf("standalone probe failed: %v", err)
+ }
+
+ poller := NewTrueNASPoller(mtp, 0, nil)
+ t.Cleanup(func() {
+ for _, byConnection := range poller.providersByOrg {
+ for _, provider := range byConnection {
+ provider.Close()
+ }
+ }
+ })
+ poller.syncConnections()
+ if poller.providersByOrg[tc.orgID][connection.ID] == nil {
+ projection, _ := json.Marshal(poller.ConnectionSummaries(tc.orgID, []config.TrueNASInstance{connection})[connection.ID])
+ t.Fatalf("standalone probe succeeded but saved provider is absent: %s", projection)
+ }
+ poller.pollAll(ctx)
+ summary := poller.ConnectionSummaries(tc.orgID, []config.TrueNASInstance{connection})[connection.ID]
+ if summary.Poll == nil || summary.Poll.LastSuccessAt == nil || summary.Poll.LastAttemptAt == nil ||
+ summary.Observed == nil || summary.Observed.Host != "storage-owner-a" || summary.Transport == nil ||
+ !summary.Transport.Connected || !hasTrueNASHostForOrg(poller, tc.orgID, "storage-owner-a") {
+ t.Fatalf("ordinary polling did not reach saved connection: %+v", summary)
+ }
+ for _, other := range []string{"default", "tenant-b", "metadata-redirect"} {
+ if other != tc.orgID && len(poller.GetCurrentRecordsForOrg(other)) != 0 {
+ t.Fatalf("resources escaped storage owner into %s", other)
+ }
+ }
+ if _, err := os.Stat(filepath.Join(mtp.BaseDataDir(), "orgs", "metadata-redirect")); !os.IsNotExist(err) {
+ t.Fatal("metadata created an unrelated tenant directory")
+ }
+ })
+ }
+}
+
+func TestTrueNASPollerStorageDiscoveryAppliesRevocationWithBrokenPeerMetadata(t *testing.T) {
+ for _, action := range []string{"disable", "remove"} {
+ t.Run(action, func(t *testing.T) {
+ server := newTrueNASMockServer(t, "revocation-owner")
+ t.Cleanup(server.Close)
+ mtp, persistence := newTestTenantPersistence(t)
+ connection := trueNASInstanceForServer(t, "revocation-id", server.URL(), true)
+ if err := persistence.SaveTrueNASConfig([]config.TrueNASInstance{connection}); err != nil {
+ t.Fatal(err)
+ }
+ poller := NewTrueNASPoller(mtp, 0, nil)
+ t.Cleanup(func() {
+ for _, byConnection := range poller.providersByOrg {
+ for _, provider := range byConnection {
+ provider.Close()
+ }
+ }
+ })
+ poller.syncConnections()
+ poller.pollAll(context.Background())
+ if !hasTrueNASHostForOrg(poller, "default", "revocation-owner") {
+ t.Fatal("initial healthy fixture did not poll")
+ }
+ before := server.RequestCount()
+ setDiscoveryOrgMetadata(t, discoveryTenant(t, mtp, "broken-peer"), "{broken")
+ next := []config.TrueNASInstance{}
+ if action == "disable" {
+ connection.Enabled = false
+ next = append(next, connection)
+ }
+ if err := persistence.SaveTrueNASConfig(next); err != nil {
+ t.Fatal(err)
+ }
+ poller.syncConnections()
+ if poller.providersByOrg["default"][connection.ID] != nil {
+ t.Fatal("unrelated org metadata blocked authoritative connection revocation")
+ }
+ poller.pollAll(context.Background())
+ if server.RequestCount() != before || len(poller.GetCurrentRecordsForOrg("default")) != 0 {
+ t.Fatal("revoked connection still polled or contributed current records")
+ }
+ if action == "remove" && poller.statusByOrg["default"][connection.ID] != nil {
+ t.Fatal("removed connection retained runtime state")
+ }
+ })
+ }
+}
+
+func TestTrueNASPollerStorageIdentitySeparatesTenantsAndTracksRemoval(t *testing.T) {
+ mtp, _ := newTestTenantPersistence(t)
+ servers := map[string]*trueNASMockServer{}
+ for _, orgID := range []string{"tenant-a", "tenant-b"} {
+ server := newTrueNASMockServer(t, orgID+"-nas")
+ t.Cleanup(server.Close)
+ servers[orgID] = server
+ connection := trueNASInstanceForServer(t, "same-local-id", server.URL(), true)
+ persistence := discoveryTenant(t, mtp, orgID)
+ if err := persistence.SaveTrueNASConfig([]config.TrueNASInstance{connection}); err != nil {
+ t.Fatal(err)
+ }
+ setDiscoveryOrgMetadata(t, persistence, `{"id":"tenant-b"}`)
+ }
+ poller := NewTrueNASPoller(mtp, 0, nil)
+ t.Cleanup(func() {
+ for _, byConnection := range poller.providersByOrg {
+ for _, provider := range byConnection {
+ provider.Close()
+ }
+ }
+ })
+ poller.syncConnections()
+ poller.pollAll(context.Background())
+ for _, orgID := range []string{"tenant-a", "tenant-b"} {
+ if !hasTrueNASHostForOrg(poller, orgID, orgID+"-nas") {
+ t.Fatalf("missing own tenant's resources: %s", orgID)
+ }
+ for _, other := range []string{"tenant-a", "tenant-b"} {
+ if other != orgID && hasTrueNASHostForOrg(poller, orgID, other+"-nas") {
+ t.Fatal("same local connection ID merged different tenant inventories")
+ }
+ }
+ }
+ retained := poller.providersByOrg["tenant-b"]["same-local-id"]
+ removedRequests := servers["tenant-a"].RequestCount()
+ if err := mtp.DeleteOrganization("tenant-a"); err != nil {
+ t.Fatal(err)
+ }
+ poller.syncConnections()
+ poller.pollAll(context.Background())
+ if len(poller.GetCurrentRecordsForOrg("tenant-a")) != 0 || len(poller.providersByOrg["tenant-a"]) != 0 ||
+ len(poller.statusByOrg["tenant-a"]) != 0 || servers["tenant-a"].RequestCount() != removedRequests {
+ t.Fatal("removed tenant retained runtime inventory or kept polling")
+ }
+ if poller.providersByOrg["tenant-b"]["same-local-id"] != retained ||
+ !hasTrueNASHostForOrg(poller, "tenant-b", "tenant-b-nas") {
+ t.Fatal("removed tenant disrupted a different storage owner's provider")
+ }
+}
+
+func TestTrueNASPollerStorageDiscoveryDoesNotReuseUnreadableConnectionConfig(t *testing.T) {
+ server := newTrueNASMockServer(t, "config-read-owner")
+ t.Cleanup(server.Close)
+ mtp, persistence := newTestTenantPersistence(t)
+ connection := trueNASInstanceForServer(t, "config-read-id", server.URL(), true)
+ if err := persistence.SaveTrueNASConfig([]config.TrueNASInstance{connection}); err != nil {
+ t.Fatal(err)
+ }
+ poller := NewTrueNASPoller(mtp, 0, nil)
+ t.Cleanup(func() {
+ for _, byConnection := range poller.providersByOrg {
+ for _, provider := range byConnection {
+ provider.Close()
+ }
+ }
+ })
+ poller.syncConnections()
+ poller.pollAll(context.Background())
+ if !hasTrueNASHostForOrg(poller, "default", "config-read-owner") {
+ t.Fatal("initial fixture did not poll")
+ }
+ before := server.RequestCount()
+ if err := os.WriteFile(filepath.Join(persistence.GetConfigDir(), "truenas.enc"), []byte("{broken"), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ poller.syncConnections()
+ poller.pollAll(context.Background())
+ if poller.providersByOrg["default"][connection.ID] != nil || server.RequestCount() != before ||
+ len(poller.GetCurrentRecordsForOrg("default")) != 0 {
+ t.Fatal("unreadable authoritative config reused old credentials or records")
+ }
+}
diff --git a/internal/monitoring/vmware_poller.go b/internal/monitoring/vmware_poller.go
index 572a40aa13..fc4a7667bd 100644
--- a/internal/monitoring/vmware_poller.go
+++ b/internal/monitoring/vmware_poller.go
@@ -193,7 +193,7 @@ func (p *VMwarePoller) syncConnections() {
return
}
- orgs, err := p.multiTenant.ListOrganizations()
+ orgIDs, err := p.multiTenant.ListOrganizationIDs()
if err != nil {
log.Warn().
Str("component", "vmware_poller").
@@ -203,15 +203,8 @@ func (p *VMwarePoller) syncConnections() {
return
}
- active := make(map[string]map[string]config.VMwareVCenterInstance, len(orgs))
- for _, org := range orgs {
- if org == nil {
- continue
- }
- orgID := strings.TrimSpace(org.ID)
- if orgID == "" {
- continue
- }
+ active := make(map[string]map[string]config.VMwareVCenterInstance, len(orgIDs))
+ for _, orgID := range orgIDs {
persistence, err := p.multiTenant.GetPersistence(orgID)
if err != nil || persistence == nil {
log.Warn().
diff --git a/internal/monitoring/vmware_poller_test.go b/internal/monitoring/vmware_poller_test.go
index 448562421d..92a3a649a4 100644
--- a/internal/monitoring/vmware_poller_test.go
+++ b/internal/monitoring/vmware_poller_test.go
@@ -3,6 +3,8 @@ package monitoring
import (
"context"
"errors"
+ "os"
+ "path/filepath"
"strings"
"testing"
"time"
@@ -481,3 +483,104 @@ func captureVMwarePollerLogs(t *testing.T) *monitoringLogCapture {
t.Helper()
return newMonitoringLogCapture(t)
}
+
+func TestVMwarePollerStorageIdentityDiscovery(t *testing.T) {
+ for _, tc := range []struct{ name, metadata, peerMetadata string }{
+ {name: "normal_missing_metadata"},
+ {name: "malformed_metadata", metadata: "{broken"},
+ {name: "missing_metadata_id", metadata: `{}`},
+ {name: "metadata_redirect", metadata: `{"id":"metadata-redirect"}`},
+ {name: "unrelated_malformed_metadata", peerMetadata: "{broken"},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ mtp, persistence := newTestTenantPersistence(t)
+ connection := config.NewVMwareVCenterInstance()
+ connection.ID = "vc-storage-id"
+ connection.Host = "fixture-vcenter"
+ connection.Username = "fixture-owner"
+ connection.Password = "fixture-password"
+ if err := persistence.SaveVMwareConfig([]config.VMwareVCenterInstance{connection}); err != nil {
+ t.Fatal(err)
+ }
+ if tc.metadata != "" {
+ setDiscoveryOrgMetadata(t, persistence, tc.metadata)
+ }
+ if tc.peerMetadata != "" {
+ setDiscoveryOrgMetadata(t, discoveryTenant(t, mtp, "broken-peer"), tc.peerMetadata)
+ }
+ poller := NewVMwarePoller(mtp, 0)
+ poller.newProvider = func(instance config.VMwareVCenterInstance) (vmwarePollerProvider, error) {
+ return vmware.NewProvider(vmware.InventorySnapshot{
+ ConnectionID: instance.ID, CollectedAt: time.Now().UTC(),
+ Hosts: []vmware.InventoryHost{{Host: "host-1", Name: "storage-owned-host"}},
+ }), nil
+ }
+ t.Cleanup(func() {
+ for _, byConnection := range poller.providersByOrg {
+ for _, provider := range byConnection {
+ provider.Close()
+ }
+ }
+ })
+ poller.syncConnections()
+ poller.pollAll(context.Background())
+ summary := poller.ConnectionSummaries("default", []config.VMwareVCenterInstance{connection})[connection.ID]
+ if summary.Poll == nil || summary.Poll.LastSuccessAt == nil || summary.Observed == nil ||
+ summary.Observed.Hosts != 1 || len(poller.GetCurrentRecordsForOrg("default")) != 1 {
+ t.Fatalf("saved VMware config was not discovered independently of metadata: %+v", summary)
+ }
+ if _, err := os.Stat(filepath.Join(mtp.BaseDataDir(), "orgs", "metadata-redirect")); !os.IsNotExist(err) {
+ t.Fatal("metadata created an unrelated tenant directory")
+ }
+ })
+ }
+}
+
+func TestVMwarePollerStorageDiscoveryAppliesRevocationWithBrokenPeerMetadata(t *testing.T) {
+ for _, action := range []string{"disable", "remove", "unreadable_config"} {
+ t.Run(action, func(t *testing.T) {
+ mtp, persistence := newTestTenantPersistence(t)
+ connection := config.NewVMwareVCenterInstance()
+ connection.ID = "vc-storage-id"
+ connection.Host = "fixture-vcenter"
+ connection.Username = "fixture-owner"
+ connection.Password = "fixture-password"
+ if err := persistence.SaveVMwareConfig([]config.VMwareVCenterInstance{connection}); err != nil {
+ t.Fatal(err)
+ }
+ poller := NewVMwarePoller(mtp, 0)
+ poller.newProvider = func(instance config.VMwareVCenterInstance) (vmwarePollerProvider, error) {
+ return vmware.NewProvider(vmware.InventorySnapshot{
+ ConnectionID: instance.ID, CollectedAt: time.Now().UTC(),
+ Hosts: []vmware.InventoryHost{{Host: "host-1", Name: "storage-owned-host"}},
+ }), nil
+ }
+ poller.syncConnections()
+ poller.pollAll(context.Background())
+ if len(poller.GetCurrentRecordsForOrg("default")) != 1 {
+ t.Fatal("initial fixture did not poll")
+ }
+ setDiscoveryOrgMetadata(t, discoveryTenant(t, mtp, "broken-peer"), "{broken")
+ switch action {
+ case "disable":
+ connection.Enabled = false
+ if err := persistence.SaveVMwareConfig([]config.VMwareVCenterInstance{connection}); err != nil {
+ t.Fatal(err)
+ }
+ case "remove":
+ if err := persistence.SaveVMwareConfig(nil); err != nil {
+ t.Fatal(err)
+ }
+ case "unreadable_config":
+ if err := os.WriteFile(filepath.Join(persistence.GetConfigDir(), "vmware.enc"), []byte("{broken"), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ }
+ poller.syncConnections()
+ poller.pollAll(context.Background())
+ if poller.providersByOrg["default"][connection.ID] != nil || len(poller.GetCurrentRecordsForOrg("default")) != 0 {
+ t.Fatal("unrelated metadata blocked fail-closed config reconciliation")
+ }
+ })
+ }
+}