Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

29.7 KB
Raw Blame
package monitoring

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"reflect"
	"strings"
	"testing"
	"testing/synctest"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/config"
	"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
	"github.com/rcourtman/pulse-go-rewrite/internal/vmware"
)

func TestVMwarePollerPollsConfiguredConnections(t *testing.T) {
	previous := vmware.IsFeatureEnabled()
	vmware.SetFeatureEnabled(true)
	t.Cleanup(func() { vmware.SetFeatureEnabled(previous) })

	mtp, persistence := newTestTenantPersistence(t)
	connection := config.NewVMwareVCenterInstance()
	connection.ID = "vc-1"
	connection.Name = "Lab VC"
	connection.Host = "vc.lab.local"
	connection.Username = "[email protected]"
	connection.Password = "super-secret"

	if err := persistence.SaveVMwareConfig([]config.VMwareVCenterInstance{connection}); err != nil {
		t.Fatalf("SaveVMwareConfig() error = %v", err)
	}

	poller := NewVMwarePoller(mtp, 20*time.Millisecond)
	poller.newProvider = func(instance config.VMwareVCenterInstance) (vmwarePollerProvider, error) {
		return vmware.NewProvider(vmware.InventorySnapshot{
			ConnectionID:   instance.ID,
			ConnectionName: instance.Name,
			VCenterHost:    instance.Host,
			CollectedAt:    time.Now().UTC(),
			Hosts: []vmware.InventoryHost{{
				Host:            "host-101",
				Name:            "esxi-01.lab.local",
				ConnectionState: "CONNECTED",
				PowerState:      "POWERED_ON",
				HostUUID:        "uuid-host-1",
				RecentTasks: []vmware.InventoryTask{{
					Task:      "task-11",
					Name:      "Reconnect host",
					State:     "running",
					StartedAt: time.Now().UTC().Add(-time.Minute),
				}},
			}},
			VMs: []vmware.InventoryVM{{
				VM:            "vm-201",
				Name:          "app-01",
				PowerState:    "POWERED_ON",
				CPUCount:      2,
				MemorySizeMiB: 4096,
				RecentEvents: []vmware.InventoryEvent{{
					Event:     "event-201",
					Type:      "VmMessageEvent",
					Message:   "Snapshot completed successfully",
					User:      "vpxuser",
					CreatedAt: time.Now().UTC().Add(-2 * time.Minute),
				}},
			}},
			Datastores: []vmware.InventoryDatastore{{
				Datastore: "datastore-11",
				Name:      "nvme-primary",
				Type:      "VMFS",
				FreeSpace: 50,
				Capacity:  100,
			}},
		}), nil
	}
	poller.Start(context.Background())
	t.Cleanup(poller.Stop)

	waitForCondition(t, 2*time.Second, func() bool {
		return len(poller.GetCurrentRecordsForOrg("default")) == 3
	}, "expected VMware poller to ingest projected records")

	records := poller.GetCurrentRecordsForOrg("default")
	if len(records) != 3 {
		t.Fatalf("expected 3 VMware records, got %d", len(records))
	}
	gotSourceIDs := make([]string, 0, len(records))
	for _, record := range records {
		gotSourceIDs = append(gotSourceIDs, record.SourceID)
	}
	wantSourceIDs := []string{
		"vc-1:host:host-101",
		"vc-1:vm:vm-201",
		"vc-1:datastore:datastore-11",
	}
	for i, want := range wantSourceIDs {
		if gotSourceIDs[i] != want {
			t.Fatalf("record %d sourceID = %q, want %q", i, gotSourceIDs[i], want)
		}
	}

	ownedSources := poller.SnapshotOwnedSourcesForOrg("default")
	if len(ownedSources) != 1 || ownedSources[0] != unifiedresources.SourceVMware {
		t.Fatalf("owned sources = %#v, want [%q]", ownedSources, unifiedresources.SourceVMware)
	}

	changes := poller.GetCurrentChangesForOrg("default")
	if len(changes) != 2 {
		t.Fatalf("expected 2 VMware activity changes, got %d", len(changes))
	}
	if changes[0].Kind != unifiedresources.ChangeActivity {
		t.Fatalf("latest VMware change kind = %q, want %q", changes[0].Kind, unifiedresources.ChangeActivity)
	}
	if changes[0].SourceAdapter != unifiedresources.AdapterVMware {
		t.Fatalf("latest VMware change source adapter = %q, want %q", changes[0].SourceAdapter, unifiedresources.AdapterVMware)
	}
	if supplemental := poller.SupplementalChanges(nil, "default"); len(supplemental) != 2 {
		t.Fatalf("expected SupplementalChanges to mirror cached VMware changes, got %d", len(supplemental))
	}

	waitForCondition(t, 2*time.Second, func() bool {
		summary, ok := poller.ConnectionSummaries("default", []config.VMwareVCenterInstance{connection})[connection.ID]
		return ok && summary.Poll != nil && summary.Poll.LastSuccessAt != nil && summary.Observed != nil
	}, "expected VMware poller to publish connection summary")

	summary := poller.ConnectionSummaries("default", []config.VMwareVCenterInstance{connection})[connection.ID]
	if summary.Poll == nil || summary.Poll.IntervalSeconds != 60 {
		t.Fatalf("expected VMware poll interval 60, got %+v", summary.Poll)
	}
	if summary.Observed == nil || summary.Observed.Hosts != 1 || summary.Observed.VMs != 1 || summary.Observed.Datastores != 1 {
		t.Fatalf("unexpected observed summary: %+v", summary.Observed)
	}
}

func TestVMwarePollerSupplementalInventoryReadyAtUsesPersistedActiveConnections(t *testing.T) {
	mtp, persistence := newTestTenantPersistence(t)

	connection := config.NewVMwareVCenterInstance()
	connection.ID = "vc-ready"
	connection.Host = "vc-ready.lab.local"
	connection.Username = "[email protected]"
	connection.Password = "secret"
	if err := persistence.SaveVMwareConfig([]config.VMwareVCenterInstance{connection}); err != nil {
		t.Fatalf("SaveVMwareConfig() error = %v", err)
	}

	poller := NewVMwarePoller(mtp, time.Second)

	if readyAt, settled := poller.SupplementalInventoryReadyAt(nil, "default"); settled || !readyAt.IsZero() {
		t.Fatalf("SupplementalInventoryReadyAt() before any attempt = (%v, %t), want (zero, false)", readyAt, settled)
	}

	attemptedAt := time.Now().UTC()
	poller.RecordConnectionTestSuccess("default", connection.ID, &vmware.InventorySummary{}, attemptedAt)

	readyAt, settled := poller.SupplementalInventoryReadyAt(nil, "default")
	if !settled {
		t.Fatal("expected readiness to settle after the first recorded attempt")
	}
	if !readyAt.Equal(attemptedAt) {
		t.Fatalf("SupplementalInventoryReadyAt() = %v, want %v", readyAt, attemptedAt)
	}
}

func TestVMwarePollerFeatureFlagGate(t *testing.T) {
	previous := vmware.IsFeatureEnabled()
	vmware.SetFeatureEnabled(false)
	t.Cleanup(func() { vmware.SetFeatureEnabled(previous) })

	mtp, persistence := newTestTenantPersistence(t)
	connection := config.NewVMwareVCenterInstance()
	connection.ID = "vc-flag-off"
	connection.Host = "vc.disabled.local"
	connection.Username = "user"
	connection.Password = "secret"

	if err := persistence.SaveVMwareConfig([]config.VMwareVCenterInstance{connection}); err != nil {
		t.Fatalf("SaveVMwareConfig() error = %v", err)
	}

	poller := NewVMwarePoller(mtp, 20*time.Millisecond)
	initialStopped := poller.stopped
	poller.Start(context.Background())

	if poller.cancel != nil {
		t.Fatal("expected Start() to be a no-op with feature flag disabled")
	}
	if poller.stopped != initialStopped {
		t.Fatal("expected stopped channel to remain unchanged when Start() is gated")
	}
	if records := poller.GetCurrentRecordsForOrg("default"); len(records) != 0 {
		t.Fatalf("expected no VMware records when feature flag is disabled, got %d", len(records))
	}
}

func TestVMwarePollerConnectionSummariesCaptureFailuresWithoutClearingObservedSummary(t *testing.T) {
	poller := NewVMwarePoller(nil, time.Minute)
	connection := config.NewVMwareVCenterInstance()
	connection.ID = "vc-1"
	connection.Host = "vc.lab.local"
	connection.Username = "[email protected]"
	connection.Password = "secret"

	successAt := time.Date(2026, 3, 31, 9, 0, 0, 0, time.UTC)
	failureAt := successAt.Add(5 * time.Minute)
	poller.RecordConnectionTestSuccess("default", connection.ID, &vmware.InventorySummary{
		Hosts:      2,
		VMs:        14,
		Datastores: 3,
		Networks:   5,
		VIRelease:  "8.0.3",
	}, successAt)
	poller.RecordConnectionTestFailure("default", connection.ID, &vmware.ConnectionError{
		Category: "permission",
		Message:  "VMware permissions are insufficient for vmware performance metrics",
	}, failureAt)

	summary := poller.ConnectionSummaries("default", []config.VMwareVCenterInstance{connection})[connection.ID]
	if summary.Poll == nil || summary.Poll.LastError == nil {
		t.Fatalf("expected poll failure summary, got %+v", summary.Poll)
	}
	if summary.Poll.ConsecutiveFailures != 1 || summary.Poll.LastError.Category != "permission" {
		t.Fatalf("unexpected poll failure details: %+v", summary.Poll)
	}
	if summary.Observed == nil || summary.Observed.VMs != 14 || summary.Observed.Networks != 5 || summary.Observed.VIRelease != "8.0.3" {
		t.Fatalf("expected observed summary to be preserved after failure, got %+v", summary.Observed)
	}
}

func TestVMwarePollerSyncRecordsProviderConstructionFailure(t *testing.T) {
	mtp, persistence := newTestTenantPersistence(t)
	connection := config.NewVMwareVCenterInstance()
	connection.ID = "vc-build-fail"
	connection.Host = "vc.build-fail.local"
	connection.Username = "[email protected]"
	connection.Password = "secret"
	if err := persistence.SaveVMwareConfig([]config.VMwareVCenterInstance{connection}); err != nil {
		t.Fatalf("SaveVMwareConfig() error = %v", err)
	}

	poller := NewVMwarePoller(mtp, time.Minute)
	poller.newProvider = func(config.VMwareVCenterInstance) (vmwarePollerProvider, error) {
		return nil, &vmware.ConnectionError{
			Category: "auth",
			Message:  "VMware credentials were rejected",
		}
	}

	poller.syncConnections()
	poller.syncConnections()

	if providers := poller.providersByOrg["default"]; len(providers) != 0 {
		t.Fatalf("expected provider construction failure to leave no providers, got %d", len(providers))
	}

	summary := poller.ConnectionSummaries("default", []config.VMwareVCenterInstance{connection})[connection.ID]
	if summary.Poll == nil || summary.Poll.LastAttemptAt == nil || summary.Poll.LastError == nil {
		t.Fatalf("expected provider construction failure in poll summary, got %+v", summary.Poll)
	}
	if summary.Poll.ConsecutiveFailures != 2 {
		t.Fatalf("consecutive failures = %d, want 2", summary.Poll.ConsecutiveFailures)
	}
	if summary.Poll.LastError.Category != "auth" || !strings.Contains(summary.Poll.LastError.Message, "credentials were rejected") {
		t.Fatalf("unexpected provider construction error summary: %+v", summary.Poll.LastError)
	}
}

type failingVMwarePollerProvider struct {
	err error
}

func (p *failingVMwarePollerProvider) Refresh(context.Context) error            { return p.err }
func (p *failingVMwarePollerProvider) Records() []unifiedresources.IngestRecord { return nil }
func (p *failingVMwarePollerProvider) ActivityChanges() []unifiedresources.ResourceChange {
	return nil
}
func (p *failingVMwarePollerProvider) Snapshot() *vmware.InventorySnapshot { return nil }
func (p *failingVMwarePollerProvider) Close()                              {}

func TestVMwarePollerLiveFailureKeepsCachedRecords(t *testing.T) {
	previous := vmware.IsFeatureEnabled()
	vmware.SetFeatureEnabled(true)
	t.Cleanup(func() { vmware.SetFeatureEnabled(previous) })

	mtp, persistence := newTestTenantPersistence(t)
	connection := config.NewVMwareVCenterInstance()
	connection.ID = "vc-live-fail"
	connection.Host = "vc.fail.local"
	connection.Username = "[email protected]"
	connection.Password = "secret"
	if err := persistence.SaveVMwareConfig([]config.VMwareVCenterInstance{connection}); err != nil {
		t.Fatalf("SaveVMwareConfig() error = %v", err)
	}

	providerCalls := 0
	poller := NewVMwarePoller(mtp, 20*time.Millisecond)
	poller.newProvider = func(instance config.VMwareVCenterInstance) (vmwarePollerProvider, error) {
		providerCalls++
		if providerCalls == 1 {
			return vmware.NewProvider(vmware.InventorySnapshot{
				ConnectionID: instance.ID,
				CollectedAt:  time.Now().UTC(),
				VIRelease:    "8.0.3",
				Hosts:        []vmware.InventoryHost{{Host: "host-1", Name: "esxi-01"}},
			}), nil
		}
		return &failingVMwarePollerProvider{
			err: errors.New("refresh vmware inventory: VMware permissions are insufficient for vmware performance metrics"),
		}, nil
	}
	poller.syncConnections()
	poller.pollAll(context.Background())

	initialRecords := poller.GetCurrentRecordsForOrg("default")
	if len(initialRecords) != 1 {
		t.Fatalf("expected initial cached VMware records, got %d", len(initialRecords))
	}

	poller.providersByOrg["default"][connection.ID] = &failingVMwarePollerProvider{
		err: &vmware.ConnectionError{Category: "permission", Message: "VMware permissions are insufficient for vmware performance metrics"},
	}
	poller.pollAll(context.Background())

	recordsAfterFailure := poller.GetCurrentRecordsForOrg("default")
	if len(recordsAfterFailure) != 1 {
		t.Fatalf("expected cached VMware records to survive live failure, got %d", len(recordsAfterFailure))
	}
	summary := poller.ConnectionSummaries("default", []config.VMwareVCenterInstance{connection})[connection.ID]
	if summary.Poll == nil || summary.Poll.LastError == nil || summary.Poll.LastError.Category != "permission" {
		t.Fatalf("expected permission poll error after live failure, got %+v", summary.Poll)
	}
}

func TestVMwarePollerPublishesDegradedObservedSummaryOnPartialSuccess(t *testing.T) {
	poller := NewVMwarePoller(nil, time.Minute)
	connection := config.NewVMwareVCenterInstance()
	connection.ID = "vc-partial"
	connection.Host = "vc.partial.local"
	connection.Username = "[email protected]"
	connection.Password = "secret"

	collectedAt := time.Date(2026, 3, 31, 10, 0, 0, 0, time.UTC)
	poller.newProvider = func(instance config.VMwareVCenterInstance) (vmwarePollerProvider, error) {
		return vmware.NewProvider(vmware.InventorySnapshot{
			ConnectionID: instance.ID,
			CollectedAt:  collectedAt,
			VIRelease:    "8.0.3",
			Hosts:        []vmware.InventoryHost{{Host: "host-101", Name: "esxi-01"}},
			EnrichmentIssues: []vmware.InventoryEnrichmentIssue{
				{Stage: "signals", EntityType: "host", EntityID: "host-101", Category: "permission", Message: "VMware permissions are insufficient for HostSystem overall status"},
				{Stage: "signals", EntityType: "vm", EntityID: "vm-201", Category: "permission", Message: "VMware permissions are insufficient for HostSystem overall status"},
				{Stage: "topology", EntityType: "vm", EntityID: "vm-201", Category: "unavailable", Message: "VMware vm guest identity is temporarily unavailable"},
			},
		}), nil
	}
	poller.providersByOrg["default"] = map[string]vmwarePollerProvider{}
	provider, err := poller.newProvider(connection)
	if err != nil {
		t.Fatalf("newProvider() error = %v", err)
	}
	poller.providersByOrg["default"][connection.ID] = provider
	poller.configsByOrg["default"] = map[string]config.VMwareVCenterInstance{connection.ID: connection}

	poller.pollAll(context.Background())

	summary := poller.ConnectionSummaries("default", []config.VMwareVCenterInstance{connection})[connection.ID]
	if summary.Poll == nil || summary.Poll.LastSuccessAt == nil || summary.Poll.LastError != nil {
		t.Fatalf("expected successful degraded poll summary, got %+v", summary.Poll)
	}
	if summary.Observed == nil || !summary.Observed.Degraded || summary.Observed.IssueCount != 3 {
		t.Fatalf("expected degraded observed summary, got %+v", summary.Observed)
	}
	if len(summary.Observed.Issues) != 2 {
		t.Fatalf("expected aggregated observed issues, got %+v", summary.Observed.Issues)
	}
	if summary.Observed.Issues[0].Occurrences != 2 || summary.Observed.Issues[0].Category != "permission" {
		t.Fatalf("expected aggregated permission issue first, got %+v", summary.Observed.Issues[0])
	}
}

type sequencedVMwarePollerProvider struct {
	snapshots []*vmware.InventorySnapshot
	index     int
	current   *vmware.InventorySnapshot
}

func (p *sequencedVMwarePollerProvider) Refresh(context.Context) error {
	if len(p.snapshots) == 0 {
		p.current = nil
		return nil
	}
	if p.index >= len(p.snapshots) {
		p.current = p.snapshots[len(p.snapshots)-1]
		return nil
	}
	p.current = p.snapshots[p.index]
	p.index++
	return nil
}

func (p *sequencedVMwarePollerProvider) Records() []unifiedresources.IngestRecord { return nil }
func (p *sequencedVMwarePollerProvider) ActivityChanges() []unifiedresources.ResourceChange {
	return nil
}
func (p *sequencedVMwarePollerProvider) Snapshot() *vmware.InventorySnapshot { return p.current }
func (p *sequencedVMwarePollerProvider) Close()                              {}

func TestVMwarePollerLogsDegradedTransitionsOnlyWhenStateChanges(t *testing.T) {
	logs := captureVMwarePollerLogs(t)
	poller := NewVMwarePoller(nil, time.Minute)
	connection := config.NewVMwareVCenterInstance()
	connection.ID = "vc-log"
	connection.Host = "vc.log.local"
	connection.Username = "[email protected]"
	connection.Password = "secret"

	issueA := vmware.InventoryEnrichmentIssue{
		Stage:    "signals",
		Category: "permission",
		Message:  "VMware permissions are insufficient for HostSystem overall status",
	}
	issueB := vmware.InventoryEnrichmentIssue{
		Stage:    "topology",
		Category: "unavailable",
		Message:  "VMware vm guest identity is temporarily unavailable",
	}
	provider := &sequencedVMwarePollerProvider{
		snapshots: []*vmware.InventorySnapshot{
			{
				ConnectionID:     connection.ID,
				CollectedAt:      time.Date(2026, 3, 31, 11, 0, 0, 0, time.UTC),
				Hosts:            []vmware.InventoryHost{{Host: "host-101", Name: "esxi-01"}},
				EnrichmentIssues: []vmware.InventoryEnrichmentIssue{issueA, issueA},
			},
			{
				ConnectionID:     connection.ID,
				CollectedAt:      time.Date(2026, 3, 31, 11, 1, 0, 0, time.UTC),
				Hosts:            []vmware.InventoryHost{{Host: "host-101", Name: "esxi-01"}},
				EnrichmentIssues: []vmware.InventoryEnrichmentIssue{issueA, issueA},
			},
			{
				ConnectionID:     connection.ID,
				CollectedAt:      time.Date(2026, 3, 31, 11, 2, 0, 0, time.UTC),
				Hosts:            []vmware.InventoryHost{{Host: "host-101", Name: "esxi-01"}},
				EnrichmentIssues: []vmware.InventoryEnrichmentIssue{issueB},
			},
			{
				ConnectionID: connection.ID,
				CollectedAt:  time.Date(2026, 3, 31, 11, 3, 0, 0, time.UTC),
				Hosts:        []vmware.InventoryHost{{Host: "host-101", Name: "esxi-01"}},
			},
		},
	}

	entry := vmwarePollerProviderEntry{
		orgID:        "default",
		connectionID: connection.ID,
		provider:     provider,
	}
	for i := 0; i < 4; i++ {
		poller.pollConnection(context.Background(), entry)
	}

	output := logs.String()
	if got := strings.Count(output, `"action":"refresh_partial"`); got != 1 {
		t.Fatalf("expected one refresh_partial warning, got %d logs: %s", got, output)
	}
	if got := strings.Count(output, `"action":"refresh_partial_changed"`); got != 1 {
		t.Fatalf("expected one refresh_partial_changed warning, got %d logs: %s", got, output)
	}
	if got := strings.Count(output, `"action":"refresh_recovered"`); got != 1 {
		t.Fatalf("expected one refresh_recovered info log, got %d logs: %s", got, output)
	}

	summary := poller.ConnectionSummaries("default", []config.VMwareVCenterInstance{connection})[connection.ID]
	if summary.Observed == nil || summary.Observed.Degraded {
		t.Fatalf("expected recovered observed summary after final healthy poll, got %+v", summary.Observed)
	}
}

func captureVMwarePollerLogs(t *testing.T) *monitoringLogCapture {
	t.Helper()
	return newMonitoringLogCapture(t)
}

func TestVMwarePollerStorageIdentityDiscovery(t *testing.T) {
	for _, tc := range []struct{ name, metadata, peerMetadata string }{
		{name: "normal_missing_metadata"},
		{name: "malformed_metadata", metadata: "{broken"},
		{name: "missing_metadata_id", metadata: `{}`},
		{name: "metadata_redirect", metadata: `{"id":"metadata-redirect"}`},
		{name: "unrelated_malformed_metadata", peerMetadata: "{broken"},
	} {
		t.Run(tc.name, func(t *testing.T) {
			mtp, persistence := newTestTenantPersistence(t)
			connection := config.NewVMwareVCenterInstance()
			connection.ID = "vc-storage-id"
			connection.Host = "fixture-vcenter"
			connection.Username = "fixture-owner"
			connection.Password = "fixture-password"
			if err := persistence.SaveVMwareConfig([]config.VMwareVCenterInstance{connection}); err != nil {
				t.Fatal(err)
			}
			if tc.metadata != "" {
				setDiscoveryOrgMetadata(t, persistence, tc.metadata)
			}
			if tc.peerMetadata != "" {
				setDiscoveryOrgMetadata(t, discoveryTenant(t, mtp, "broken-peer"), tc.peerMetadata)
			}
			poller := NewVMwarePoller(mtp, 0)
			poller.newProvider = func(instance config.VMwareVCenterInstance) (vmwarePollerProvider, error) {
				return vmware.NewProvider(vmware.InventorySnapshot{
					ConnectionID: instance.ID, CollectedAt: time.Now().UTC(),
					Hosts: []vmware.InventoryHost{{Host: "host-1", Name: "storage-owned-host"}},
				}), nil
			}
			t.Cleanup(func() {
				for _, byConnection := range poller.providersByOrg {
					for _, provider := range byConnection {
						provider.Close()
					}
				}
			})
			poller.syncConnections()
			poller.pollAll(context.Background())
			summary := poller.ConnectionSummaries("default", []config.VMwareVCenterInstance{connection})[connection.ID]
			if summary.Poll == nil || summary.Poll.LastSuccessAt == nil || summary.Observed == nil ||
				summary.Observed.Hosts != 1 || len(poller.GetCurrentRecordsForOrg("default")) != 1 {
				t.Fatalf("saved VMware config was not discovered independently of metadata: %+v", summary)
			}
			if _, err := os.Stat(filepath.Join(mtp.BaseDataDir(), "orgs", "metadata-redirect")); !os.IsNotExist(err) {
				t.Fatal("metadata created an unrelated tenant directory")
			}
		})
	}
}

func TestVMwarePollerStorageDiscoveryAppliesRevocationWithBrokenPeerMetadata(t *testing.T) {
	for _, action := range []string{"disable", "remove", "unreadable_config"} {
		t.Run(action, func(t *testing.T) {
			mtp, persistence := newTestTenantPersistence(t)
			connection := config.NewVMwareVCenterInstance()
			connection.ID = "vc-storage-id"
			connection.Host = "fixture-vcenter"
			connection.Username = "fixture-owner"
			connection.Password = "fixture-password"
			if err := persistence.SaveVMwareConfig([]config.VMwareVCenterInstance{connection}); err != nil {
				t.Fatal(err)
			}
			poller := NewVMwarePoller(mtp, 0)
			poller.newProvider = func(instance config.VMwareVCenterInstance) (vmwarePollerProvider, error) {
				return vmware.NewProvider(vmware.InventorySnapshot{
					ConnectionID: instance.ID, CollectedAt: time.Now().UTC(),
					Hosts: []vmware.InventoryHost{{Host: "host-1", Name: "storage-owned-host"}},
				}), nil
			}
			poller.syncConnections()
			poller.pollAll(context.Background())
			if len(poller.GetCurrentRecordsForOrg("default")) != 1 {
				t.Fatal("initial fixture did not poll")
			}
			setDiscoveryOrgMetadata(t, discoveryTenant(t, mtp, "broken-peer"), "{broken")
			switch action {
			case "disable":
				connection.Enabled = false
				if err := persistence.SaveVMwareConfig([]config.VMwareVCenterInstance{connection}); err != nil {
					t.Fatal(err)
				}
			case "remove":
				if err := persistence.SaveVMwareConfig(nil); err != nil {
					t.Fatal(err)
				}
			case "unreadable_config":
				if err := os.WriteFile(filepath.Join(persistence.GetConfigDir(), "vmware.enc"), []byte("{broken"), 0o600); err != nil {
					t.Fatal(err)
				}
			}
			poller.syncConnections()
			poller.pollAll(context.Background())
			if poller.providersByOrg["default"][connection.ID] != nil || len(poller.GetCurrentRecordsForOrg("default")) != 0 {
				t.Fatal("unrelated metadata blocked fail-closed config reconciliation")
			}
		})
	}
}

// Parsing never removes the whole-refresh bound or copies arbitrary environment
// text into a warning. Tests do not contact a vCenter or alter a live poller.
func testVMwarePollTimeoutConfiguration(t *testing.T) {
	for _, tc := range []struct {
		raw     string
		want    time.Duration
		warning bool
	}{
		{"", 20 * time.Second, false}, {" \t", 20 * time.Second, false},
		{"120s", 2 * time.Minute, false}, {" 2m ", 2 * time.Minute, false},
		{"250ms", 250 * time.Millisecond, false},
		{"0", 20 * time.Second, true}, {"-1s", 20 * time.Second, true},
		{"synthetic-private-value", 20 * time.Second, true},
		{"9999999999999999999h", 20 * time.Second, true},
	} {
		t.Run(fmt.Sprintf("case-%d", len(tc.raw))+"/"+tc.want.String(), func(t *testing.T) {
			t.Setenv("PULSE_VMWARE_POLL_TIMEOUT", tc.raw)
			captured := captureVMwarePollerLogs(t)
			if got := vmwarePollTimeout(); got != tc.want {
				t.Fatalf("timeout=%s want %s", got, tc.want)
			}
			text := captured.String()
			if strings.Contains(text, "synthetic-private-value") {
				t.Fatal("warning copied environment contents")
			}
			if strings.Contains(text, "Invalid PULSE_VMWARE_POLL_TIMEOUT") != tc.warning {
				t.Fatalf("warning=%q want warning=%t", text, tc.warning)
			}
		})
	}
}

type timedVMwarePollerProvider struct {
	vmwarePollerProvider
	delay             time.Duration
	active, maxActive *int
	order             *[]string
	id                string
}

func (p *timedVMwarePollerProvider) Refresh(ctx context.Context) error {
	*p.active++
	defer func() { *p.active-- }()
	if *p.active > *p.maxActive {
		*p.maxActive = *p.active
	}
	*p.order = append(*p.order, p.id)
	select {
	case <-ctx.Done():
		return ctx.Err()
	case <-time.After(p.delay):
		return p.vmwarePollerProvider.Refresh(ctx)
	}
}

// A real twenty-one-second refresh crosses the unchanged default twenty-second
// outer bound under Go's virtual clock. The configured budget admits it, but
// never lengthens the parent deadline, overlaps connections or drops the last
// complete inventory on failure. Production provider -> registry rows are the
// payloads consumed by the browser proof; these are not installed observations.
func testVMwarePollRefreshBudgetAndProjection(t *testing.T) {
	previous := vmware.IsFeatureEnabled()
	vmware.SetFeatureEnabled(true)
	t.Cleanup(func() { vmware.SetFeatureEnabled(previous) })
	t.Setenv("PULSE_VMWARE_POLL_TIMEOUT", "")
	synctest.Test(t, func(t *testing.T) {
		connection := config.NewVMwareVCenterInstance()
		connection.ID, connection.Name, connection.Host = "budget-vcenter", "Budget vCenter", "vcenter.example.invalid"
		poller := NewVMwarePoller(nil, time.Minute)
		snapshot := func(name string) vmware.InventorySnapshot {
			return vmware.InventorySnapshot{ConnectionID: connection.ID, ConnectionName: connection.Name, VCenterHost: connection.Host, CollectedAt: time.Now().UTC(), VIRelease: "8.0.3",
				VMs: []vmware.InventoryVM{{VM: "vm-101", Name: name, PowerState: "POWERED_ON", CPUCount: 2, MemorySizeMiB: 4096}},
			}
		}
		entry := vmwarePollerProviderEntry{orgID: "default", connectionID: connection.ID, provider: vmware.NewProvider(snapshot("app-before-timeout"))}
		poller.pollConnection(context.Background(), entry)
		original := poller.ConnectionSummaries("default", []config.VMwareVCenterInstance{connection})[connection.ID]
		oldRows := poller.GetCurrentRecords()
		if original.Poll.LastSuccessAt == nil || len(oldRows) != 1 {
			t.Fatal("fixture did not seed a complete inventory")
		}
		project := func(phase string) {
			adapter := unifiedresources.NewMonitorAdapter(nil)
			adapter.PopulateSupplementalRecords(unifiedresources.SourceVMware, poller.GetCurrentRecords())
			payload, err := json.Marshal(struct {
				Phase     string                      `json:"phase"`
				Summary   VMwareConnectionSummary     `json:"summary"`
				Resources []unifiedresources.Resource `json:"resources"`
			}{phase, poller.ConnectionSummaries("default", []config.VMwareVCenterInstance{connection})[connection.ID], adapter.GetAll()})
			if err != nil {
				t.Fatal(err)
			}
			t.Logf("VMWARE_BUDGET_PROJECTION_JSON=%s", payload)
		}
		active, maximum := 0, 0
		order := []string{}
		slow := &timedVMwarePollerProvider{vmwarePollerProvider: vmware.NewProvider(snapshot("app-after-timeout")), delay: 21 * time.Second, active: &active, maxActive: &maximum, order: &order, id: "primary"}
		entry.provider = slow
		started := time.Now()
		poller.pollConnection(context.Background(), entry)
		failed := poller.ConnectionSummaries("default", []config.VMwareVCenterInstance{connection})[connection.ID]
		if time.Since(started) != 20*time.Second || failed.Poll.ConsecutiveFailures != 1 || failed.Poll.LastError == nil || !failed.Poll.LastSuccessAt.Equal(*original.Poll.LastSuccessAt) || !reflect.DeepEqual(oldRows, poller.GetCurrentRecords()) {
			t.Fatalf("default timeout dropped/renewed accepted inventory or lost failure: %+v", failed)
		}
		project("default-timeout-retains-inventory")
		t.Setenv("PULSE_VMWARE_POLL_TIMEOUT", "30s")
		started = time.Now()
		poller.pollConnection(context.Background(), entry)
		recovered := poller.ConnectionSummaries("default", []config.VMwareVCenterInstance{connection})[connection.ID]
		if time.Since(started) != 21*time.Second || recovered.Poll.ConsecutiveFailures != 0 || recovered.Poll.LastError != nil || !recovered.Poll.LastSuccessAt.After(*original.Poll.LastSuccessAt) || reflect.DeepEqual(oldRows, poller.GetCurrentRecords()) {
			t.Fatalf("configured budget did not recover a complete refresh: %+v", recovered)
		}
		project("configured-refresh-recovers")
		acceptedRows := poller.GetCurrentRecords()
		parent, cancel := context.WithTimeout(context.Background(), time.Second)
		started = time.Now()
		poller.pollConnection(parent, entry)
		cancel()
		if time.Since(started) != time.Second || !reflect.DeepEqual(acceptedRows, poller.GetCurrentRecords()) {
			t.Fatal("configured refresh ignored parent or dropped the accepted inventory")
		}
		cancelled := poller.ConnectionSummaries("default", []config.VMwareVCenterInstance{connection})[connection.ID]
		if cancelled.Poll.LastError == nil || !strings.Contains(cancelled.Poll.LastError.Message, context.DeadlineExceeded.Error()) || !cancelled.Poll.LastSuccessAt.Equal(*recovered.Poll.LastSuccessAt) {
			t.Fatalf("parent deadline renewed success: %+v", cancelled)
		}
		project("short-parent-retains-inventory")
		// Two configured connections use the production serial dispatcher. A longer
		// allowed refresh is not permission for a second simultaneous one.
		sibling := &timedVMwarePollerProvider{vmwarePollerProvider: vmware.NewProvider(snapshot("sibling-app")), delay: time.Second, active: &active, maxActive: &maximum, order: &order, id: "sibling"}
		poller.providersByOrg["default"] = map[string]vmwarePollerProvider{connection.ID: slow, "sibling": sibling}
		order = nil
		poller.pollAll(context.Background())
		if active != 0 || maximum != 1 || !reflect.DeepEqual(order, []string{"primary", "sibling"}) {
			t.Fatalf("polls overlap or lose serial order: active/max/order=%d/%d/%v", active, maximum, order)
		}
	})
}