Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

49.0 KB
Raw Blame
package vmware

import (
	"context"
	"fmt"
	"net/url"
	"sort"
	"strings"
	"sync"
)

type vcenterVMIdentity struct {
	BIOSUUID     string `json:"bios_uuid"`
	InstanceUUID string `json:"instance_uuid"`
}

type vcenterVMHardwareInfo struct {
	Version             string                       `json:"version"`
	UpgradePolicy       string                       `json:"upgrade_policy"`
	UpgradeVersion      string                       `json:"upgrade_version"`
	UpgradeStatus       string                       `json:"upgrade_status"`
	UpgradeErrorMessage *vcenterVMLocalizableMessage `json:"upgrade_error"`
}

type vcenterVMBootInfo struct {
	Type            string `json:"type"`
	EFILegacyBoot   *bool  `json:"efi_legacy_boot"`
	NetworkProtocol string `json:"network_protocol"`
	Delay           *int64 `json:"delay"`
	Retry           *bool  `json:"retry"`
	RetryDelay      *int64 `json:"retry_delay"`
	EnterSetupMode  *bool  `json:"enter_setup_mode"`
}

type vcenterVMBootDeviceInfo struct {
	Type  string   `json:"type"`
	NIC   string   `json:"nic"`
	Disks []string `json:"disks"`
}

type vcenterVMCPUInfo struct {
	Count            *int64 `json:"count"`
	CoresPerSocket   *int64 `json:"cores_per_socket"`
	HotAddEnabled    *bool  `json:"hot_add_enabled"`
	HotRemoveEnabled *bool  `json:"hot_remove_enabled"`
}

type vcenterVMMemoryInfo struct {
	SizeMiB                *int64 `json:"size_mib"`
	HotAddEnabled          *bool  `json:"hot_add_enabled"`
	HotAddIncrementSizeMiB *int64 `json:"hot_add_increment_size_mib"`
	HotAddLimitMiB         *int64 `json:"hot_add_limit_mib"`
}

type vcenterVMInfo struct {
	GuestOS            string                    `json:"guest_os"`
	Identity           *vcenterVMIdentity        `json:"identity"`
	PowerState         string                    `json:"power_state"`
	InstantCloneFrozen *bool                     `json:"instant_clone_frozen"`
	Hardware           *vcenterVMHardwareInfo    `json:"hardware"`
	Boot               *vcenterVMBootInfo        `json:"boot"`
	BootDevices        []vcenterVMBootDeviceInfo `json:"boot_devices"`
	CPU                *vcenterVMCPUInfo         `json:"cpu"`
	Memory             *vcenterVMMemoryInfo      `json:"memory"`
}

type vcenterVMGuestIdentity struct {
	Family   string `json:"family"`
	HostName string `json:"host_name"`
	IPAddr   string `json:"ip_address"`
}

type vcenterVMLocalizableMessage struct {
	ID             string `json:"id"`
	DefaultMessage string `json:"default_message"`
	Localized      string `json:"localized"`
}

type vcenterVMToolsGuestRebootStatus struct {
	RebootRequested      *bool    `json:"reboot_requested"`
	RequestingComponents []string `json:"requesting_components"`
	RequestTimestamp     string   `json:"request_timestamp"`
}

type vcenterVMToolsInfo struct {
	AutoUpdateSupported *bool                            `json:"auto_update_supported"`
	InstallAttemptCount *int64                           `json:"install_attempt_count"`
	Error               *vcenterVMLocalizableMessage     `json:"error"`
	VersionNumber       *int64                           `json:"version_number"`
	Version             string                           `json:"version"`
	UpgradePolicy       string                           `json:"upgrade_policy"`
	VersionStatus       string                           `json:"version_status"`
	InstallType         string                           `json:"install_type"`
	RunState            string                           `json:"run_state"`
	GuestRebootStatus   *vcenterVMToolsGuestRebootStatus `json:"guest_reboot_status"`
}

type vcenterVMHardwareEthernetSummary struct {
	NIC string `json:"nic"`
}

type vcenterVMHardwareEthernetBacking struct {
	Type                  string `json:"type"`
	Network               string `json:"network"`
	NetworkName           string `json:"network_name"`
	HostDevice            string `json:"host_device"`
	DistributedSwitchUUID string `json:"distributed_switch_uuid"`
	DistributedPort       string `json:"distributed_port"`
	OpaqueNetworkType     string `json:"opaque_network_type"`
	OpaqueNetworkID       string `json:"opaque_network_id"`
}

type vcenterVMHardwareEthernetInfo struct {
	Label              string                           `json:"label"`
	Type               string                           `json:"type"`
	UPTCompatibility   bool                             `json:"upt_compatibility_enabled"`
	UPTV2Compatibility bool                             `json:"upt_v2_compatibility_enabled"`
	MACType            string                           `json:"mac_type"`
	MACAddress         string                           `json:"mac_address"`
	PCISlotNumber      *int64                           `json:"pci_slot_number"`
	WakeOnLANEnabled   bool                             `json:"wake_on_lan_enabled"`
	Backing            vcenterVMHardwareEthernetBacking `json:"backing"`
	State              string                           `json:"state"`
	StartConnected     bool                             `json:"start_connected"`
	AllowGuestControl  bool                             `json:"allow_guest_control"`
}

type vcenterVMHardwareDiskSummary struct {
	Disk string `json:"disk"`
}

type vcenterVMHardwareDiskBacking struct {
	Type     string `json:"type"`
	VMDKFile string `json:"vmdk_file"`
}

type vcenterVMHardwareDiskIDEAddress struct {
	Primary *bool `json:"primary"`
	Master  *bool `json:"master"`
}

type vcenterVMHardwareDiskBusUnitAddress struct {
	Bus  *int64 `json:"bus"`
	Unit *int64 `json:"unit"`
}

type vcenterVMHardwareDiskInfo struct {
	Label    string                               `json:"label"`
	Type     string                               `json:"type"`
	IDE      *vcenterVMHardwareDiskIDEAddress     `json:"ide"`
	SCSI     *vcenterVMHardwareDiskBusUnitAddress `json:"scsi"`
	SATA     *vcenterVMHardwareDiskBusUnitAddress `json:"sata"`
	NVME     *vcenterVMHardwareDiskBusUnitAddress `json:"nvme"`
	Backing  vcenterVMHardwareDiskBacking         `json:"backing"`
	Capacity *int64                               `json:"capacity"`
}

type viJSONVirtualMachineRuntimeInfo struct {
	Host *viJSONReference `json:"host"`
}

type viJSONDatastoreSummary struct {
	Accessible         *bool  `json:"accessible"`
	MultipleHostAccess *bool  `json:"multipleHostAccess"`
	MaintenanceMode    string `json:"maintenanceMode"`
	URL                string `json:"url"`
}

type vmwarePlacement struct {
	DatacenterID        string
	DatacenterName      string
	ComputeResourceID   string
	ComputeResourceName string
	ClusterID           string
	ClusterName         string
	FolderID            string
	FolderName          string
}

type vmwareTopologyCache struct {
	mu sync.Mutex

	nameLoaded map[string]bool
	names      map[string]string

	parentLoaded map[string]bool
	parents      map[string]*viJSONReference

	ownerLoaded map[string]bool
	owners      map[string]*viJSONReference

	placementLoaded map[string]bool
	placements      map[string]vmwarePlacement
}

func newVMwareTopologyCache() *vmwareTopologyCache {
	return &vmwareTopologyCache{
		nameLoaded:      make(map[string]bool),
		names:           make(map[string]string),
		parentLoaded:    make(map[string]bool),
		parents:         make(map[string]*viJSONReference),
		ownerLoaded:     make(map[string]bool),
		owners:          make(map[string]*viJSONReference),
		placementLoaded: make(map[string]bool),
		placements:      make(map[string]vmwarePlacement),
	}
}

func (c *Client) enrichInventoryTopology(
	ctx context.Context,
	automationSessionID string,
	release string,
	sessionID string,
	snapshot *InventorySnapshot,
) ([]InventoryEnrichmentIssue, error) {
	if snapshot == nil {
		return nil, nil
	}

	hostNamesByID := make(map[string]string, len(snapshot.Hosts))
	for _, host := range snapshot.Hosts {
		hostNamesByID[strings.TrimSpace(host.Host)] = firstNonEmptyTrimmed(host.Name, host.Host)
	}
	vmNamesByID := make(map[string]string, len(snapshot.VMs))
	for _, vm := range snapshot.VMs {
		vmNamesByID[strings.TrimSpace(vm.VM)] = firstNonEmptyTrimmed(vm.Name, vm.VM)
	}
	datastoreNamesByID := make(map[string]string, len(snapshot.Datastores))
	for _, datastore := range snapshot.Datastores {
		datastoreNamesByID[strings.TrimSpace(datastore.Datastore)] = firstNonEmptyTrimmed(datastore.Name, datastore.Datastore)
	}

	var issues []InventoryEnrichmentIssue
	clusters, err := c.collectClusterInventory(ctx, automationSessionID)
	if issue, ok := classifyInventoryEnrichmentIssue("topology", "cluster", "", err); ok {
		issues = append(issues, *issue)
	} else if err != nil && !isAutomationNotFound(err) && !isAutomationUnavailable(err) {
		return nil, err
	}
	snapshot.Clusters = clusters
	clusterDetailsByID := inventoryClusterDetailsByID(clusters)

	cache := newVMwareTopologyCache()
	sem := make(chan struct{}, vmwareSignalEnrichmentConcurrency)
	var wg sync.WaitGroup
	var firstErr error
	var firstErrMu sync.Mutex
	var issuesMu sync.Mutex

	recordIssues := func(values []InventoryEnrichmentIssue) {
		if len(values) == 0 {
			return
		}
		issuesMu.Lock()
		issues = append(issues, values...)
		issuesMu.Unlock()
	}

	run := func(fn func() error) {
		wg.Add(1)
		go func() {
			defer wg.Done()
			sem <- struct{}{}
			defer func() { <-sem }()

			if err := fn(); err != nil {
				firstErrMu.Lock()
				if firstErr == nil {
					firstErr = err
				}
				firstErrMu.Unlock()
			}
		}()
	}

	for i := range snapshot.Hosts {
		i := i
		run(func() error {
			host, hostIssues, err := c.enrichHostTopology(ctx, release, sessionID, snapshot.Hosts[i], cache, datastoreNamesByID, clusterDetailsByID)
			if err != nil {
				return err
			}
			recordIssues(hostIssues)
			snapshot.Hosts[i] = host
			return nil
		})
	}

	for i := range snapshot.VMs {
		i := i
		run(func() error {
			vm, vmIssues, err := c.enrichVMTopology(
				ctx,
				automationSessionID,
				release,
				sessionID,
				snapshot.VMs[i],
				cache,
				hostNamesByID,
				datastoreNamesByID,
				clusterDetailsByID,
			)
			if err != nil {
				return err
			}
			recordIssues(vmIssues)
			snapshot.VMs[i] = vm
			return nil
		})
	}

	for i := range snapshot.Datastores {
		i := i
		run(func() error {
			datastore, datastoreIssues, err := c.enrichDatastoreTopology(ctx, release, sessionID, snapshot.Datastores[i], cache, hostNamesByID, vmNamesByID)
			if err != nil {
				return err
			}
			recordIssues(datastoreIssues)
			snapshot.Datastores[i] = datastore
			return nil
		})
	}

	for i := range snapshot.Networks {
		i := i
		run(func() error {
			network, networkIssues, err := c.enrichNetworkTopology(ctx, release, sessionID, snapshot.Networks[i], cache, hostNamesByID, vmNamesByID)
			if err != nil {
				return err
			}
			recordIssues(networkIssues)
			snapshot.Networks[i] = network
			return nil
		})
	}

	wg.Wait()

	firstErrMu.Lock()
	err = firstErr
	firstErrMu.Unlock()
	if err != nil {
		return nil, err
	}
	sort.Slice(issues, func(i, j int) bool {
		return inventoryEnrichmentIssueSortKey(issues[i]) < inventoryEnrichmentIssueSortKey(issues[j])
	})
	return issues, nil
}

func (c *Client) enrichHostTopology(
	ctx context.Context,
	release string,
	sessionID string,
	host InventoryHost,
	cache *vmwareTopologyCache,
	datastoreNamesByID map[string]string,
	clusterDetailsByID map[string]InventoryCluster,
) (InventoryHost, []InventoryEnrichmentIssue, error) {
	var issues []InventoryEnrichmentIssue
	recordIssue := func(issue *InventoryEnrichmentIssue) {
		if issue != nil {
			issues = append(issues, *issue)
		}
	}

	ref := viJSONReference{Type: "HostSystem", Value: strings.TrimSpace(host.Host)}
	placement, err := cache.resolvePlacement(ctx, c, release, sessionID, ref)
	if issue, ok := classifyInventoryEnrichmentIssue("topology", "host", host.Host, err); ok {
		recordIssue(issue)
	} else if err != nil && !isVIJSONNotFound(err) {
		return host, nil, err
	}
	applyPlacementToHost(&host, placement)
	applyClusterServicesToHost(&host, clusterDetailsByID)

	datastoreRefs, err := c.collectEntityReferenceList(ctx, release, sessionID, "HostSystem", host.Host, "datastore", "host datastore attachments")
	if issue, ok := classifyInventoryEnrichmentIssue("topology", "host", host.Host, err); ok {
		recordIssue(issue)
	} else if err != nil && !isVIJSONNotFound(err) {
		return host, nil, err
	}
	host.DatastoreIDs = idsForReferences(datastoreRefs)
	host.DatastoreNames = namesForReferences(datastoreRefs, datastoreNamesByID)

	return host, issues, nil
}

func (c *Client) enrichVMTopology(
	ctx context.Context,
	automationSessionID string,
	release string,
	sessionID string,
	vm InventoryVM,
	cache *vmwareTopologyCache,
	hostNamesByID map[string]string,
	datastoreNamesByID map[string]string,
	clusterDetailsByID map[string]InventoryCluster,
) (InventoryVM, []InventoryEnrichmentIssue, error) {
	var issues []InventoryEnrichmentIssue
	recordIssue := func(issue *InventoryEnrichmentIssue) {
		if issue != nil {
			issues = append(issues, *issue)
		}
	}

	vmInfo, err := c.collectVMAutomationInfo(ctx, automationSessionID, vm.VM)
	if issue, ok := classifyInventoryEnrichmentIssue("topology", "vm", vm.VM, err); ok {
		recordIssue(issue)
	} else if err != nil && !isAutomationNotFound(err) {
		return vm, nil, err
	}
	if vmInfo != nil && vmInfo.Identity != nil {
		vm.InstanceUUID = strings.TrimSpace(vmInfo.Identity.InstanceUUID)
		vm.BIOSUUID = strings.TrimSpace(vmInfo.Identity.BIOSUUID)
	}
	if vmInfo != nil && vmInfo.CPU != nil && vmInfo.CPU.Count != nil && *vmInfo.CPU.Count > 0 {
		vm.CPUCount = int(*vmInfo.CPU.Count)
	}
	if vmInfo != nil && vmInfo.Memory != nil && vmInfo.Memory.SizeMiB != nil && *vmInfo.Memory.SizeMiB > 0 {
		vm.MemorySizeMiB = *vmInfo.Memory.SizeMiB
	}
	vm.Hardware = inventoryVMHardwareFromInfo(vmInfo)

	guestIdentity, err := c.collectVMGuestIdentity(ctx, automationSessionID, vm.VM)
	if issue, ok := classifyInventoryEnrichmentIssue("topology", "vm", vm.VM, err); ok {
		recordIssue(issue)
	} else if err != nil && !isAutomationNotFound(err) && !isAutomationUnavailable(err) {
		return vm, nil, err
	}
	if guestIdentity != nil {
		vm.GuestOSFamily = strings.TrimSpace(guestIdentity.Family)
		vm.GuestHostname = strings.TrimSpace(guestIdentity.HostName)
		if ip := strings.TrimSpace(guestIdentity.IPAddr); ip != "" {
			vm.GuestIPAddresses = []string{ip}
		}
	}

	tools, err := c.collectVMTools(ctx, automationSessionID, vm.VM)
	if issue, ok := classifyInventoryEnrichmentIssue("topology", "vm", vm.VM, err); ok {
		recordIssue(issue)
	} else if err != nil && !isAutomationNotFound(err) && !isAutomationUnavailable(err) {
		return vm, nil, err
	}
	vm.Tools = tools

	networkAdapters, err := c.collectVMEthernetAdapters(ctx, automationSessionID, vm.VM)
	if issue, ok := classifyInventoryEnrichmentIssue("topology", "vm", vm.VM, err); ok {
		recordIssue(issue)
	} else if err != nil && !isAutomationNotFound(err) && !isAutomationUnavailable(err) {
		return vm, nil, err
	}
	vm.NetworkAdapters = networkAdapters

	virtualDisks, err := c.collectVMHardwareDisks(ctx, automationSessionID, vm.VM)
	if issue, ok := classifyInventoryEnrichmentIssue("topology", "vm", vm.VM, err); ok {
		recordIssue(issue)
	} else if err != nil && !isAutomationNotFound(err) && !isAutomationUnavailable(err) {
		return vm, nil, err
	}
	vm.VirtualDisks = virtualDisks

	var placement vmwarePlacement

	parentRef, err := c.collectEntityReference(ctx, release, sessionID, "VirtualMachine", vm.VM, "parent", "vm parent placement")
	if issue, ok := classifyInventoryEnrichmentIssue("topology", "vm", vm.VM, err); ok {
		recordIssue(issue)
	} else if err != nil && !isVIJSONNotFound(err) {
		return vm, nil, err
	}
	if parentRef != nil {
		parentPlacement, err := cache.resolvePlacement(ctx, c, release, sessionID, *parentRef)
		if issue, ok := classifyInventoryEnrichmentIssue("topology", "vm", vm.VM, err); ok {
			recordIssue(issue)
		} else if err != nil && !isVIJSONNotFound(err) {
			return vm, nil, err
		}
		mergePlacement(&placement, parentPlacement)
	}

	runtimeHostRef, err := c.collectVMRuntimeHostReference(ctx, release, sessionID, vm.VM)
	if issue, ok := classifyInventoryEnrichmentIssue("topology", "vm", vm.VM, err); ok {
		recordIssue(issue)
	} else if err != nil && !isVIJSONNotFound(err) {
		return vm, nil, err
	}
	if runtimeHostRef != nil {
		vm.RuntimeHostID = strings.TrimSpace(runtimeHostRef.Value)
		vm.RuntimeHostName = firstNonEmptyTrimmed(hostNamesByID[vm.RuntimeHostID], vm.RuntimeHostID)
		hostPlacement, err := cache.resolvePlacement(ctx, c, release, sessionID, *runtimeHostRef)
		if issue, ok := classifyInventoryEnrichmentIssue("topology", "vm", vm.VM, err); ok {
			recordIssue(issue)
		} else if err != nil && !isVIJSONNotFound(err) {
			return vm, nil, err
		}
		mergePlacement(&placement, hostPlacement)
	}

	resourcePoolRef, err := c.collectEntityReference(ctx, release, sessionID, "VirtualMachine", vm.VM, "resourcePool", "vm resource pool")
	if issue, ok := classifyInventoryEnrichmentIssue("topology", "vm", vm.VM, err); ok {
		recordIssue(issue)
	} else if err != nil && !isVIJSONNotFound(err) {
		return vm, nil, err
	}
	if resourcePoolRef != nil {
		vm.ResourcePoolID = strings.TrimSpace(resourcePoolRef.Value)
		resourcePoolName, err := cache.resolveName(ctx, c, release, sessionID, *resourcePoolRef)
		if issue, ok := classifyInventoryEnrichmentIssue("topology", "vm", vm.VM, err); ok {
			recordIssue(issue)
		} else if err != nil && !isVIJSONNotFound(err) {
			return vm, nil, err
		}
		vm.ResourcePoolName = firstNonEmptyTrimmed(resourcePoolName, vm.ResourcePoolID)

		ownerRef, err := cache.resolveResourcePoolOwner(ctx, c, release, sessionID, *resourcePoolRef)
		if issue, ok := classifyInventoryEnrichmentIssue("topology", "vm", vm.VM, err); ok {
			recordIssue(issue)
		} else if err != nil && !isVIJSONNotFound(err) {
			return vm, nil, err
		}
		if ownerRef != nil {
			ownerPlacement, err := cache.resolvePlacement(ctx, c, release, sessionID, *ownerRef)
			if issue, ok := classifyInventoryEnrichmentIssue("topology", "vm", vm.VM, err); ok {
				recordIssue(issue)
			} else if err != nil && !isVIJSONNotFound(err) {
				return vm, nil, err
			}
			mergePlacement(&placement, ownerPlacement)
		}
	}

	datastoreRefs, err := c.collectEntityReferenceList(ctx, release, sessionID, "VirtualMachine", vm.VM, "datastore", "vm datastore attachments")
	if issue, ok := classifyInventoryEnrichmentIssue("topology", "vm", vm.VM, err); ok {
		recordIssue(issue)
	} else if err != nil && !isVIJSONNotFound(err) {
		return vm, nil, err
	}
	vm.DatastoreIDs = idsForReferences(datastoreRefs)
	vm.DatastoreNames = namesForReferences(datastoreRefs, datastoreNamesByID)

	applyPlacementToVM(&vm, placement)
	applyClusterServicesToVM(&vm, clusterDetailsByID)
	return vm, issues, nil
}

func (c *Client) collectClusterInventory(
	ctx context.Context,
	automationSessionID string,
) ([]InventoryCluster, error) {
	if strings.TrimSpace(automationSessionID) == "" {
		return nil, nil
	}

	var payload []InventoryCluster
	if err := c.listAutomationResources(ctx, automationSessionID, "/api/vcenter/cluster", "cluster inventory", &payload); err != nil {
		return nil, err
	}

	clusters := make([]InventoryCluster, 0, len(payload))
	for _, cluster := range payload {
		id := strings.TrimSpace(cluster.Cluster)
		if id == "" {
			continue
		}
		clusters = append(clusters, InventoryCluster{
			Cluster:    id,
			Name:       strings.TrimSpace(cluster.Name),
			HAEnabled:  cloneBoolPointer(cluster.HAEnabled),
			DRSEnabled: cloneBoolPointer(cluster.DRSEnabled),
		})
	}
	sort.Slice(clusters, func(i, j int) bool {
		return vmwareSortKey(clusters[i].Cluster, clusters[i].Name) < vmwareSortKey(clusters[j].Cluster, clusters[j].Name)
	})
	return clusters, nil
}

func (c *Client) enrichNetworkTopology(
	ctx context.Context,
	release string,
	sessionID string,
	network InventoryNetwork,
	cache *vmwareTopologyCache,
	hostNamesByID map[string]string,
	vmNamesByID map[string]string,
) (InventoryNetwork, []InventoryEnrichmentIssue, error) {
	var issues []InventoryEnrichmentIssue
	recordIssue := func(issue *InventoryEnrichmentIssue) {
		if issue != nil {
			issues = append(issues, *issue)
		}
	}

	ref := viJSONReference{Type: "Network", Value: strings.TrimSpace(network.Network)}
	placement, err := cache.resolvePlacement(ctx, c, release, sessionID, ref)
	if issue, ok := classifyInventoryEnrichmentIssue("topology", "network", network.Network, err); ok {
		recordIssue(issue)
	} else if err != nil && !isVIJSONNotFound(err) {
		return network, nil, err
	}
	applyPlacementToNetwork(&network, placement)

	hostRefs, err := c.collectEntityReferenceList(ctx, release, sessionID, "Network", network.Network, "host", "network host attachments")
	if issue, ok := classifyInventoryEnrichmentIssue("topology", "network", network.Network, err); ok {
		recordIssue(issue)
	} else if err != nil && !isVIJSONNotFound(err) {
		return network, nil, err
	}
	network.HostIDs = idsForReferences(hostRefs)
	network.HostNames = namesForReferences(hostRefs, hostNamesByID)

	vmRefs, err := c.collectEntityReferenceList(ctx, release, sessionID, "Network", network.Network, "vm", "network vm attachments")
	if issue, ok := classifyInventoryEnrichmentIssue("topology", "network", network.Network, err); ok {
		recordIssue(issue)
	} else if err != nil && !isVIJSONNotFound(err) {
		return network, nil, err
	}
	network.VMIDs = idsForReferences(vmRefs)
	network.VMNames = namesForReferences(vmRefs, vmNamesByID)

	return network, issues, nil
}

func (c *Client) enrichDatastoreTopology(
	ctx context.Context,
	release string,
	sessionID string,
	datastore InventoryDatastore,
	cache *vmwareTopologyCache,
	hostNamesByID map[string]string,
	vmNamesByID map[string]string,
) (InventoryDatastore, []InventoryEnrichmentIssue, error) {
	var issues []InventoryEnrichmentIssue
	recordIssue := func(issue *InventoryEnrichmentIssue) {
		if issue != nil {
			issues = append(issues, *issue)
		}
	}

	ref := viJSONReference{Type: "Datastore", Value: strings.TrimSpace(datastore.Datastore)}
	placement, err := cache.resolvePlacement(ctx, c, release, sessionID, ref)
	if issue, ok := classifyInventoryEnrichmentIssue("topology", "storage", datastore.Datastore, err); ok {
		recordIssue(issue)
	} else if err != nil && !isVIJSONNotFound(err) {
		return datastore, nil, err
	}
	applyPlacementToDatastore(&datastore, placement)

	summary, err := c.collectDatastoreSummary(ctx, release, sessionID, datastore.Datastore)
	if issue, ok := classifyInventoryEnrichmentIssue("topology", "storage", datastore.Datastore, err); ok {
		recordIssue(issue)
	} else if err != nil && !isVIJSONNotFound(err) {
		return datastore, nil, err
	}
	if summary != nil {
		datastore.Accessible = summary.Accessible
		datastore.MultipleHostAccess = summary.MultipleHostAccess
		datastore.MaintenanceMode = strings.TrimSpace(summary.MaintenanceMode)
		datastore.URL = strings.TrimSpace(summary.URL)
	}

	hostRefs, err := c.collectEntityReferenceList(ctx, release, sessionID, "Datastore", datastore.Datastore, "host", "datastore host attachments")
	if issue, ok := classifyInventoryEnrichmentIssue("topology", "storage", datastore.Datastore, err); ok {
		recordIssue(issue)
	} else if err != nil && !isVIJSONNotFound(err) {
		return datastore, nil, err
	}
	datastore.HostIDs = idsForReferences(hostRefs)
	datastore.HostNames = namesForReferences(hostRefs, hostNamesByID)

	vmRefs, err := c.collectEntityReferenceList(ctx, release, sessionID, "Datastore", datastore.Datastore, "vm", "datastore vm attachments")
	if issue, ok := classifyInventoryEnrichmentIssue("topology", "storage", datastore.Datastore, err); ok {
		recordIssue(issue)
	} else if err != nil && !isVIJSONNotFound(err) {
		return datastore, nil, err
	}
	datastore.VMIDs = idsForReferences(vmRefs)
	datastore.VMNames = namesForReferences(vmRefs, vmNamesByID)

	return datastore, issues, nil
}

func (c *Client) collectVMAutomationInfo(
	ctx context.Context,
	automationSessionID string,
	vmID string,
) (*vcenterVMInfo, error) {
	vmID = strings.TrimSpace(vmID)
	if vmID == "" || strings.TrimSpace(automationSessionID) == "" {
		return nil, nil
	}
	var payload vcenterVMInfo
	path := fmt.Sprintf("/api/vcenter/vm/%s", url.PathEscape(vmID))
	if err := c.getAutomationJSON(ctx, automationSessionID, path, "vm detail", &payload); err != nil {
		return nil, err
	}
	return &payload, nil
}

func inventoryVMHardwareFromInfo(info *vcenterVMInfo) *InventoryVMHardware {
	if info == nil {
		return nil
	}
	hardware := &InventoryVMHardware{
		GuestOS:            strings.TrimSpace(info.GuestOS),
		InstantCloneFrozen: cloneBoolPointer(info.InstantCloneFrozen),
	}
	if info.Hardware != nil {
		hardware.Version = strings.TrimSpace(info.Hardware.Version)
		hardware.UpgradePolicy = strings.TrimSpace(info.Hardware.UpgradePolicy)
		hardware.UpgradeVersion = strings.TrimSpace(info.Hardware.UpgradeVersion)
		hardware.UpgradeStatus = strings.TrimSpace(info.Hardware.UpgradeStatus)
		hardware.UpgradeErrorMessage = localizableMessageText(info.Hardware.UpgradeErrorMessage)
	}
	if info.Boot != nil {
		hardware.BootType = strings.TrimSpace(info.Boot.Type)
		hardware.EFILegacyBoot = cloneBoolPointer(info.Boot.EFILegacyBoot)
		hardware.BootNetworkProtocol = strings.TrimSpace(info.Boot.NetworkProtocol)
		hardware.BootDelayMilliseconds = cloneInt64Pointer(info.Boot.Delay)
		hardware.BootRetry = cloneBoolPointer(info.Boot.Retry)
		hardware.BootRetryDelayMilliseconds = cloneInt64Pointer(info.Boot.RetryDelay)
		hardware.EnterSetupMode = cloneBoolPointer(info.Boot.EnterSetupMode)
	}
	hardware.BootDevices = inventoryVMBootDevicesFromInfo(info.BootDevices)
	if info.CPU != nil {
		hardware.CPUCoresPerSocket = cloneInt64Pointer(info.CPU.CoresPerSocket)
		hardware.CPUHotAddEnabled = cloneBoolPointer(info.CPU.HotAddEnabled)
		hardware.CPUHotRemoveEnabled = cloneBoolPointer(info.CPU.HotRemoveEnabled)
	}
	if info.Memory != nil {
		hardware.MemoryHotAddEnabled = cloneBoolPointer(info.Memory.HotAddEnabled)
		hardware.MemoryHotAddIncrementMiB = cloneInt64Pointer(info.Memory.HotAddIncrementSizeMiB)
		hardware.MemoryHotAddLimitMiB = cloneInt64Pointer(info.Memory.HotAddLimitMiB)
	}
	if inventoryVMHardwareEmpty(hardware) {
		return nil
	}
	return hardware
}

func inventoryVMBootDevicesFromInfo(devices []vcenterVMBootDeviceInfo) []InventoryVMBootDevice {
	if len(devices) == 0 {
		return nil
	}
	out := make([]InventoryVMBootDevice, 0, len(devices))
	for _, device := range devices {
		out = append(out, InventoryVMBootDevice{
			Type:  strings.TrimSpace(device.Type),
			NIC:   strings.TrimSpace(device.NIC),
			Disks: cloneStringSlice(device.Disks),
		})
	}
	return out
}

func inventoryVMHardwareEmpty(hardware *InventoryVMHardware) bool {
	return hardware == nil ||
		(hardware.GuestOS == "" &&
			hardware.InstantCloneFrozen == nil &&
			hardware.Version == "" &&
			hardware.UpgradePolicy == "" &&
			hardware.UpgradeVersion == "" &&
			hardware.UpgradeStatus == "" &&
			hardware.UpgradeErrorMessage == "" &&
			hardware.BootType == "" &&
			hardware.EFILegacyBoot == nil &&
			hardware.BootNetworkProtocol == "" &&
			hardware.BootDelayMilliseconds == nil &&
			hardware.BootRetry == nil &&
			hardware.BootRetryDelayMilliseconds == nil &&
			hardware.EnterSetupMode == nil &&
			len(hardware.BootDevices) == 0 &&
			hardware.CPUCoresPerSocket == nil &&
			hardware.CPUHotAddEnabled == nil &&
			hardware.CPUHotRemoveEnabled == nil &&
			hardware.MemoryHotAddEnabled == nil &&
			hardware.MemoryHotAddIncrementMiB == nil &&
			hardware.MemoryHotAddLimitMiB == nil)
}

func (c *Client) collectVMGuestIdentity(
	ctx context.Context,
	automationSessionID string,
	vmID string,
) (*vcenterVMGuestIdentity, error) {
	vmID = strings.TrimSpace(vmID)
	if vmID == "" || strings.TrimSpace(automationSessionID) == "" {
		return nil, nil
	}
	var payload vcenterVMGuestIdentity
	path := fmt.Sprintf("/api/vcenter/vm/%s/guest/identity", vmID)
	if err := c.getAutomationJSON(ctx, automationSessionID, path, "vm guest identity", &payload); err != nil {
		return nil, err
	}
	return &payload, nil
}

// vcenterVMGuestLocalFilesystemInfo mirrors vSphere Automation REST
// `VmGuestLocalFilesystemInfo`. The endpoint returns a JSON map keyed by
// the filesystem mount-point string. Documented at
// https://developer.vmware.com/apis/vsphere-automation/latest/vcenter/api/vcenter/vm/vm/guest/local-filesystem/get/
// Fields:
//   - capacity / free_space: integer bytes (required by the schema).
//   - filesystem: optional string (e.g. "ntfs", "ext4").
//   - mappings: optional `VirtualDiskMapping` array tying the filesystem
//     back to a virtual disk; not yet consumed here.
type vcenterVMGuestLocalFilesystemInfo struct {
	Capacity   int64  `json:"capacity"`
	FreeSpace  int64  `json:"free_space"`
	Filesystem string `json:"filesystem,omitempty"`
}

// collectVMGuestLocalFilesystem fetches `/api/vcenter/vm/{vm}/guest/local-filesystem`
// and aggregates capacity / used bytes across every reported guest
// filesystem. Returns ok=false when the VM has no guest filesystem data
// (Tools not running, VM powered off, filesystem list empty); the caller
// should not mutate metrics in that case. A 503 from vCenter ("guest
// agent not running") is surfaced as an "unavailable" ConnectionError so
// the enrichment classifier treats it as a soft issue rather than a hard
// collection failure.
func (c *Client) collectVMGuestLocalFilesystem(
	ctx context.Context,
	automationSessionID string,
	vmID string,
) (totalBytes int64, usedBytes int64, percent float64, ok bool, err error) {
	vmID = strings.TrimSpace(vmID)
	if vmID == "" || strings.TrimSpace(automationSessionID) == "" {
		return 0, 0, 0, false, nil
	}
	var payload map[string]vcenterVMGuestLocalFilesystemInfo
	path := fmt.Sprintf("/api/vcenter/vm/%s/guest/local-filesystem", url.PathEscape(vmID))
	if err := c.getAutomationJSON(ctx, automationSessionID, path, "vm guest local filesystem", &payload); err != nil {
		return 0, 0, 0, false, err
	}
	if len(payload) == 0 {
		return 0, 0, 0, false, nil
	}
	for _, info := range payload {
		if info.Capacity <= 0 {
			continue
		}
		totalBytes += info.Capacity
		used := info.Capacity - info.FreeSpace
		if used < 0 {
			used = 0
		}
		usedBytes += used
	}
	if totalBytes <= 0 {
		return 0, 0, 0, false, nil
	}
	percent = float64(usedBytes) / float64(totalBytes) * 100
	if percent < 0 {
		percent = 0
	} else if percent > 100 {
		percent = 100
	}
	return totalBytes, usedBytes, percent, true, nil
}

func (c *Client) collectVMTools(
	ctx context.Context,
	automationSessionID string,
	vmID string,
) (*InventoryVMTools, error) {
	vmID = strings.TrimSpace(vmID)
	if vmID == "" || strings.TrimSpace(automationSessionID) == "" {
		return nil, nil
	}
	var payload vcenterVMToolsInfo
	path := fmt.Sprintf("/api/vcenter/vm/%s/tools", url.PathEscape(vmID))
	if err := c.getAutomationJSON(ctx, automationSessionID, path, "vm tools", &payload); err != nil {
		return nil, err
	}
	tools := &InventoryVMTools{
		AutoUpdateSupported: cloneBoolPointer(payload.AutoUpdateSupported),
		InstallAttemptCount: cloneInt64Pointer(payload.InstallAttemptCount),
		ErrorMessage:        localizableMessageText(payload.Error),
		VersionNumber:       cloneInt64Pointer(payload.VersionNumber),
		Version:             strings.TrimSpace(payload.Version),
		UpgradePolicy:       strings.TrimSpace(payload.UpgradePolicy),
		VersionStatus:       strings.TrimSpace(payload.VersionStatus),
		InstallType:         strings.TrimSpace(payload.InstallType),
		RunState:            strings.TrimSpace(payload.RunState),
	}
	if payload.GuestRebootStatus != nil {
		tools.GuestRebootRequested = cloneBoolPointer(payload.GuestRebootStatus.RebootRequested)
		tools.GuestRebootComponents = cloneStringSlice(payload.GuestRebootStatus.RequestingComponents)
		tools.GuestRebootRequestTime = strings.TrimSpace(payload.GuestRebootStatus.RequestTimestamp)
	}
	return tools, nil
}

func (c *Client) collectVMEthernetAdapters(
	ctx context.Context,
	automationSessionID string,
	vmID string,
) ([]InventoryVMNetworkAdapter, error) {
	vmID = strings.TrimSpace(vmID)
	if vmID == "" || strings.TrimSpace(automationSessionID) == "" {
		return nil, nil
	}

	escapedVMID := url.PathEscape(vmID)
	var summaries []vcenterVMHardwareEthernetSummary
	path := fmt.Sprintf("/api/vcenter/vm/%s/hardware/ethernet", escapedVMID)
	if err := c.getAutomationJSON(ctx, automationSessionID, path, "vm ethernet adapter list", &summaries); err != nil {
		return nil, err
	}
	if len(summaries) == 0 {
		return nil, nil
	}

	adapters := make([]InventoryVMNetworkAdapter, 0, len(summaries))
	for _, summary := range summaries {
		nic := strings.TrimSpace(summary.NIC)
		if nic == "" {
			continue
		}
		var info vcenterVMHardwareEthernetInfo
		path := fmt.Sprintf("/api/vcenter/vm/%s/hardware/ethernet/%s", escapedVMID, url.PathEscape(nic))
		if err := c.getAutomationJSON(ctx, automationSessionID, path, "vm ethernet adapter", &info); err != nil {
			return nil, err
		}
		adapters = append(adapters, InventoryVMNetworkAdapter{
			NIC:                   nic,
			Label:                 strings.TrimSpace(info.Label),
			Type:                  strings.TrimSpace(info.Type),
			MACType:               strings.TrimSpace(info.MACType),
			MACAddress:            strings.TrimSpace(info.MACAddress),
			PCISlotNumber:         cloneInt64Pointer(info.PCISlotNumber),
			BackingType:           strings.TrimSpace(info.Backing.Type),
			NetworkID:             strings.TrimSpace(info.Backing.Network),
			NetworkName:           strings.TrimSpace(info.Backing.NetworkName),
			DistributedSwitchUUID: strings.TrimSpace(info.Backing.DistributedSwitchUUID),
			DistributedPort:       strings.TrimSpace(info.Backing.DistributedPort),
			OpaqueNetworkType:     strings.TrimSpace(info.Backing.OpaqueNetworkType),
			OpaqueNetworkID:       strings.TrimSpace(info.Backing.OpaqueNetworkID),
			HostDevice:            strings.TrimSpace(info.Backing.HostDevice),
			State:                 strings.TrimSpace(info.State),
			StartConnected:        info.StartConnected,
			AllowGuestControl:     info.AllowGuestControl,
			WakeOnLANEnabled:      info.WakeOnLANEnabled,
			UPTCompatibility:      info.UPTCompatibility,
			UPTV2Compatibility:    info.UPTV2Compatibility,
		})
	}
	return adapters, nil
}

func (c *Client) collectVMHardwareDisks(
	ctx context.Context,
	automationSessionID string,
	vmID string,
) ([]InventoryVMVirtualDisk, error) {
	vmID = strings.TrimSpace(vmID)
	if vmID == "" || strings.TrimSpace(automationSessionID) == "" {
		return nil, nil
	}

	escapedVMID := url.PathEscape(vmID)
	var summaries []vcenterVMHardwareDiskSummary
	path := fmt.Sprintf("/api/vcenter/vm/%s/hardware/disk", escapedVMID)
	if err := c.getAutomationJSON(ctx, automationSessionID, path, "vm hardware disk list", &summaries); err != nil {
		return nil, err
	}
	if len(summaries) == 0 {
		return nil, nil
	}

	disks := make([]InventoryVMVirtualDisk, 0, len(summaries))
	for _, summary := range summaries {
		diskID := strings.TrimSpace(summary.Disk)
		if diskID == "" {
			continue
		}
		var info vcenterVMHardwareDiskInfo
		path := fmt.Sprintf("/api/vcenter/vm/%s/hardware/disk/%s", escapedVMID, url.PathEscape(diskID))
		if err := c.getAutomationJSON(ctx, automationSessionID, path, "vm hardware disk", &info); err != nil {
			return nil, err
		}
		disk := InventoryVMVirtualDisk{
			Disk:          diskID,
			Label:         strings.TrimSpace(info.Label),
			Type:          strings.TrimSpace(info.Type),
			BackingType:   strings.TrimSpace(info.Backing.Type),
			VMDKFile:      strings.TrimSpace(info.Backing.VMDKFile),
			DatastoreName: vmdkDatastoreName(info.Backing.VMDKFile),
			CapacityBytes: cloneInt64Pointer(info.Capacity),
		}
		if info.IDE != nil {
			disk.IDEPrimary = cloneBoolPointer(info.IDE.Primary)
			disk.IDEMaster = cloneBoolPointer(info.IDE.Master)
		}
		if info.SCSI != nil {
			disk.SCSIBus = cloneInt64Pointer(info.SCSI.Bus)
			disk.SCSIUnit = cloneInt64Pointer(info.SCSI.Unit)
		}
		if info.SATA != nil {
			disk.SATABus = cloneInt64Pointer(info.SATA.Bus)
			disk.SATAUnit = cloneInt64Pointer(info.SATA.Unit)
		}
		if info.NVME != nil {
			disk.NVMEBus = cloneInt64Pointer(info.NVME.Bus)
			disk.NVMEUnit = cloneInt64Pointer(info.NVME.Unit)
		}
		disks = append(disks, disk)
	}
	sort.Slice(disks, func(i, j int) bool {
		return vmwareSortKey(disks[i].Disk, disks[i].Label) < vmwareSortKey(disks[j].Disk, disks[j].Label)
	})
	return disks, nil
}

func (c *Client) collectVMRuntimeHostReference(
	ctx context.Context,
	release string,
	sessionID string,
	vmID string,
) (*viJSONReference, error) {
	vmID = strings.TrimSpace(vmID)
	if vmID == "" {
		return nil, nil
	}
	var runtime viJSONVirtualMachineRuntimeInfo
	path := fmt.Sprintf("/sdk/vim25/%s/VirtualMachine/%s/runtime", release, vmID)
	if err := c.getVIJSONJSON(ctx, sessionID, path, "vm runtime placement", &runtime); err != nil {
		return nil, err
	}
	return cloneVIJSONReference(runtime.Host), nil
}

func (c *Client) collectDatastoreSummary(
	ctx context.Context,
	release string,
	sessionID string,
	datastoreID string,
) (*viJSONDatastoreSummary, error) {
	datastoreID = strings.TrimSpace(datastoreID)
	if datastoreID == "" {
		return nil, nil
	}
	var summary viJSONDatastoreSummary
	path := fmt.Sprintf("/sdk/vim25/%s/Datastore/%s/summary", release, datastoreID)
	if err := c.getVIJSONJSON(ctx, sessionID, path, "datastore summary", &summary); err != nil {
		return nil, err
	}
	return &summary, nil
}

func vmdkDatastoreName(vmdkFile string) string {
	value := strings.TrimSpace(vmdkFile)
	if !strings.HasPrefix(value, "[") {
		return ""
	}
	end := strings.Index(value, "]")
	if end <= 1 {
		return ""
	}
	return strings.TrimSpace(value[1:end])
}

func localizableMessageText(message *vcenterVMLocalizableMessage) string {
	if message == nil {
		return ""
	}
	return firstNonEmptyTrimmed(message.Localized, message.DefaultMessage, message.ID)
}

func (c *Client) collectEntityReference(
	ctx context.Context,
	release string,
	sessionID string,
	managedType string,
	managedObjectID string,
	property string,
	label string,
) (*viJSONReference, error) {
	managedObjectID = strings.TrimSpace(managedObjectID)
	if managedObjectID == "" {
		return nil, nil
	}
	var ref *viJSONReference
	path := fmt.Sprintf("/sdk/vim25/%s/%s/%s/%s", release, managedType, managedObjectID, property)
	if err := c.getVIJSONJSON(ctx, sessionID, path, label, &ref); err != nil {
		return nil, err
	}
	return cloneVIJSONReference(ref), nil
}

func (c *Client) collectEntityReferenceList(
	ctx context.Context,
	release string,
	sessionID string,
	managedType string,
	managedObjectID string,
	property string,
	label string,
) ([]viJSONReference, error) {
	managedObjectID = strings.TrimSpace(managedObjectID)
	if managedObjectID == "" {
		return nil, nil
	}
	var refs []viJSONReference
	path := fmt.Sprintf("/sdk/vim25/%s/%s/%s/%s", release, managedType, managedObjectID, property)
	if err := c.getVIJSONJSON(ctx, sessionID, path, label, &refs); err != nil {
		return nil, err
	}
	return cloneVIJSONReferences(refs), nil
}

func (cache *vmwareTopologyCache) resolvePlacement(
	ctx context.Context,
	client *Client,
	release string,
	sessionID string,
	ref viJSONReference,
) (vmwarePlacement, error) {
	key := managedObjectKey(ref.Type, ref.Value)
	if key == "" {
		return vmwarePlacement{}, nil
	}

	cache.mu.Lock()
	if cache.placementLoaded[key] {
		placement := cache.placements[key]
		cache.mu.Unlock()
		return placement, nil
	}
	cache.mu.Unlock()

	placement := vmwarePlacement{}
	name, err := cache.resolveName(ctx, client, release, sessionID, ref)
	if err != nil && !isVIJSONNotFound(err) {
		return vmwarePlacement{}, err
	}
	name = firstNonEmptyTrimmed(name, ref.Value)

	switch strings.TrimSpace(ref.Type) {
	case "Datacenter":
		placement.DatacenterID = strings.TrimSpace(ref.Value)
		placement.DatacenterName = name
	case "Folder":
		placement.FolderID = strings.TrimSpace(ref.Value)
		placement.FolderName = name
	case "ClusterComputeResource":
		placement.ClusterID = strings.TrimSpace(ref.Value)
		placement.ClusterName = name
		placement.ComputeResourceID = strings.TrimSpace(ref.Value)
		placement.ComputeResourceName = name
	case "ComputeResource":
		placement.ComputeResourceID = strings.TrimSpace(ref.Value)
		placement.ComputeResourceName = name
	}

	parentRef, err := cache.resolveParent(ctx, client, release, sessionID, ref)
	if err != nil && !isVIJSONNotFound(err) {
		return vmwarePlacement{}, err
	}
	if parentRef != nil {
		parentPlacement, err := cache.resolvePlacement(ctx, client, release, sessionID, *parentRef)
		if err != nil && !isVIJSONNotFound(err) {
			return vmwarePlacement{}, err
		}
		mergePlacement(&placement, parentPlacement)
	}

	cache.mu.Lock()
	cache.placementLoaded[key] = true
	cache.placements[key] = placement
	cache.mu.Unlock()
	return placement, nil
}

func (cache *vmwareTopologyCache) resolveName(
	ctx context.Context,
	client *Client,
	release string,
	sessionID string,
	ref viJSONReference,
) (string, error) {
	key := managedObjectKey(ref.Type, ref.Value)
	if key == "" {
		return "", nil
	}

	cache.mu.Lock()
	if cache.nameLoaded[key] {
		name := cache.names[key]
		cache.mu.Unlock()
		return name, nil
	}
	cache.mu.Unlock()

	var name string
	path := fmt.Sprintf("/sdk/vim25/%s/%s/%s/name", release, ref.Type, ref.Value)
	if err := client.getVIJSONJSON(ctx, sessionID, path, strings.ToLower(strings.TrimSpace(ref.Type))+" name", &name); err != nil {
		return "", err
	}
	name = strings.TrimSpace(name)

	cache.mu.Lock()
	cache.nameLoaded[key] = true
	cache.names[key] = name
	cache.mu.Unlock()
	return name, nil
}

func (cache *vmwareTopologyCache) resolveParent(
	ctx context.Context,
	client *Client,
	release string,
	sessionID string,
	ref viJSONReference,
) (*viJSONReference, error) {
	key := managedObjectKey(ref.Type, ref.Value)
	if key == "" {
		return nil, nil
	}

	cache.mu.Lock()
	if cache.parentLoaded[key] {
		parent := cloneVIJSONReference(cache.parents[key])
		cache.mu.Unlock()
		return parent, nil
	}
	cache.mu.Unlock()

	parent, err := client.collectEntityReference(
		ctx,
		release,
		sessionID,
		strings.TrimSpace(ref.Type),
		strings.TrimSpace(ref.Value),
		"parent",
		strings.ToLower(strings.TrimSpace(ref.Type))+" parent",
	)
	if err != nil {
		return nil, err
	}

	cache.mu.Lock()
	cache.parentLoaded[key] = true
	cache.parents[key] = cloneVIJSONReference(parent)
	cache.mu.Unlock()
	return parent, nil
}

func (cache *vmwareTopologyCache) resolveResourcePoolOwner(
	ctx context.Context,
	client *Client,
	release string,
	sessionID string,
	ref viJSONReference,
) (*viJSONReference, error) {
	key := managedObjectKey(ref.Type, ref.Value)
	if key == "" {
		return nil, nil
	}

	cache.mu.Lock()
	if cache.ownerLoaded[key] {
		owner := cloneVIJSONReference(cache.owners[key])
		cache.mu.Unlock()
		return owner, nil
	}
	cache.mu.Unlock()

	owner, err := client.collectEntityReference(
		ctx,
		release,
		sessionID,
		"ResourcePool",
		strings.TrimSpace(ref.Value),
		"owner",
		"resource pool owner",
	)
	if err != nil {
		return nil, err
	}

	cache.mu.Lock()
	cache.ownerLoaded[key] = true
	cache.owners[key] = cloneVIJSONReference(owner)
	cache.mu.Unlock()
	return owner, nil
}

func applyPlacementToHost(host *InventoryHost, placement vmwarePlacement) {
	if host == nil {
		return
	}
	host.DatacenterID = firstNonEmptyTrimmed(host.DatacenterID, placement.DatacenterID)
	host.DatacenterName = firstNonEmptyTrimmed(host.DatacenterName, placement.DatacenterName)
	host.ComputeResourceID = firstNonEmptyTrimmed(host.ComputeResourceID, placement.ComputeResourceID)
	host.ComputeResourceName = firstNonEmptyTrimmed(host.ComputeResourceName, placement.ComputeResourceName)
	host.ClusterID = firstNonEmptyTrimmed(host.ClusterID, placement.ClusterID)
	host.ClusterName = firstNonEmptyTrimmed(host.ClusterName, placement.ClusterName)
	host.FolderID = firstNonEmptyTrimmed(host.FolderID, placement.FolderID)
	host.FolderName = firstNonEmptyTrimmed(host.FolderName, placement.FolderName)
}

func applyClusterServicesToHost(host *InventoryHost, clustersByID map[string]InventoryCluster) {
	if host == nil || len(clustersByID) == 0 {
		return
	}
	cluster, ok := clustersByID[strings.TrimSpace(host.ClusterID)]
	if !ok {
		return
	}
	host.ClusterName = firstNonEmptyTrimmed(host.ClusterName, cluster.Name)
	if cluster.HAEnabled != nil {
		host.ClusterHAEnabled = cloneBoolPointer(cluster.HAEnabled)
	}
	if cluster.DRSEnabled != nil {
		host.ClusterDRSEnabled = cloneBoolPointer(cluster.DRSEnabled)
	}
}

func applyPlacementToVM(vm *InventoryVM, placement vmwarePlacement) {
	if vm == nil {
		return
	}
	vm.DatacenterID = firstNonEmptyTrimmed(vm.DatacenterID, placement.DatacenterID)
	vm.DatacenterName = firstNonEmptyTrimmed(vm.DatacenterName, placement.DatacenterName)
	vm.ComputeResourceID = firstNonEmptyTrimmed(vm.ComputeResourceID, placement.ComputeResourceID)
	vm.ComputeResourceName = firstNonEmptyTrimmed(vm.ComputeResourceName, placement.ComputeResourceName)
	vm.ClusterID = firstNonEmptyTrimmed(vm.ClusterID, placement.ClusterID)
	vm.ClusterName = firstNonEmptyTrimmed(vm.ClusterName, placement.ClusterName)
	vm.FolderID = firstNonEmptyTrimmed(vm.FolderID, placement.FolderID)
	vm.FolderName = firstNonEmptyTrimmed(vm.FolderName, placement.FolderName)
}

func applyClusterServicesToVM(vm *InventoryVM, clustersByID map[string]InventoryCluster) {
	if vm == nil || len(clustersByID) == 0 {
		return
	}
	cluster, ok := clustersByID[strings.TrimSpace(vm.ClusterID)]
	if !ok {
		return
	}
	vm.ClusterName = firstNonEmptyTrimmed(vm.ClusterName, cluster.Name)
	if cluster.HAEnabled != nil {
		vm.ClusterHAEnabled = cloneBoolPointer(cluster.HAEnabled)
	}
	if cluster.DRSEnabled != nil {
		vm.ClusterDRSEnabled = cloneBoolPointer(cluster.DRSEnabled)
	}
}

func applyPlacementToDatastore(datastore *InventoryDatastore, placement vmwarePlacement) {
	if datastore == nil {
		return
	}
	datastore.DatacenterID = firstNonEmptyTrimmed(datastore.DatacenterID, placement.DatacenterID)
	datastore.DatacenterName = firstNonEmptyTrimmed(datastore.DatacenterName, placement.DatacenterName)
	datastore.FolderID = firstNonEmptyTrimmed(datastore.FolderID, placement.FolderID)
	datastore.FolderName = firstNonEmptyTrimmed(datastore.FolderName, placement.FolderName)
}

func applyPlacementToNetwork(network *InventoryNetwork, placement vmwarePlacement) {
	if network == nil {
		return
	}
	network.DatacenterID = firstNonEmptyTrimmed(network.DatacenterID, placement.DatacenterID)
	network.DatacenterName = firstNonEmptyTrimmed(network.DatacenterName, placement.DatacenterName)
	network.FolderID = firstNonEmptyTrimmed(network.FolderID, placement.FolderID)
	network.FolderName = firstNonEmptyTrimmed(network.FolderName, placement.FolderName)
}

func mergePlacement(dst *vmwarePlacement, src vmwarePlacement) {
	if dst == nil {
		return
	}
	if dst.DatacenterID == "" {
		dst.DatacenterID = strings.TrimSpace(src.DatacenterID)
	}
	if dst.DatacenterName == "" {
		dst.DatacenterName = strings.TrimSpace(src.DatacenterName)
	}
	if dst.ComputeResourceID == "" {
		dst.ComputeResourceID = strings.TrimSpace(src.ComputeResourceID)
	}
	if dst.ComputeResourceName == "" {
		dst.ComputeResourceName = strings.TrimSpace(src.ComputeResourceName)
	}
	if dst.ClusterID == "" {
		dst.ClusterID = strings.TrimSpace(src.ClusterID)
	}
	if dst.ClusterName == "" {
		dst.ClusterName = strings.TrimSpace(src.ClusterName)
	}
	if dst.FolderID == "" {
		dst.FolderID = strings.TrimSpace(src.FolderID)
	}
	if dst.FolderName == "" {
		dst.FolderName = strings.TrimSpace(src.FolderName)
	}
}

func inventoryClusterDetailsByID(clusters []InventoryCluster) map[string]InventoryCluster {
	if len(clusters) == 0 {
		return nil
	}
	out := make(map[string]InventoryCluster, len(clusters))
	for _, cluster := range clusters {
		id := strings.TrimSpace(cluster.Cluster)
		if id == "" {
			continue
		}
		out[id] = InventoryCluster{
			Cluster:    id,
			Name:       strings.TrimSpace(cluster.Name),
			HAEnabled:  cloneBoolPointer(cluster.HAEnabled),
			DRSEnabled: cloneBoolPointer(cluster.DRSEnabled),
		}
	}
	return out
}

func idsForReferences(refs []viJSONReference) []string {
	ids := make([]string, 0, len(refs))
	for _, ref := range refs {
		if id := strings.TrimSpace(ref.Value); id != "" {
			ids = append(ids, id)
		}
	}
	return uniqueSortedTrimmedStrings(ids)
}

func namesForReferences(refs []viJSONReference, namesByID map[string]string) []string {
	names := make([]string, 0, len(refs))
	for _, ref := range refs {
		id := strings.TrimSpace(ref.Value)
		if id == "" {
			continue
		}
		name := firstNonEmptyTrimmed(namesByID[id], id)
		if name != "" {
			names = append(names, name)
		}
	}
	return uniqueSortedTrimmedStrings(names)
}

func cloneVIJSONReference(in *viJSONReference) *viJSONReference {
	if in == nil {
		return nil
	}
	out := *in
	out.Type = strings.TrimSpace(out.Type)
	out.Value = strings.TrimSpace(out.Value)
	return &out
}

func cloneVIJSONReferences(in []viJSONReference) []viJSONReference {
	if in == nil {
		return nil
	}
	out := make([]viJSONReference, 0, len(in))
	for _, ref := range in {
		trimmed := viJSONReference{
			Type:  strings.TrimSpace(ref.Type),
			Value: strings.TrimSpace(ref.Value),
		}
		if trimmed.Value == "" {
			continue
		}
		out = append(out, trimmed)
	}
	return out
}

func managedObjectKey(managedType, managedObjectID string) string {
	managedType = strings.TrimSpace(managedType)
	managedObjectID = strings.TrimSpace(managedObjectID)
	if managedType == "" || managedObjectID == "" {
		return ""
	}
	return managedType + "/" + managedObjectID
}

func uniqueSortedTrimmedStrings(values []string) []string {
	if len(values) == 0 {
		return nil
	}
	seen := make(map[string]struct{}, len(values))
	out := make([]string, 0, len(values))
	for _, value := range values {
		trimmed := strings.TrimSpace(value)
		if trimmed == "" {
			continue
		}
		key := strings.ToLower(trimmed)
		if _, ok := seen[key]; ok {
			continue
		}
		seen[key] = struct{}{}
		out = append(out, trimmed)
	}
	sort.Strings(out)
	if len(out) == 0 {
		return nil
	}
	return out
}

func isAutomationNotFound(err error) bool {
	connectionErr, ok := err.(*ConnectionError)
	return ok && connectionErr.Category == "not_found"
}

func isAutomationUnavailable(err error) bool {
	connectionErr, ok := err.(*ConnectionError)
	return ok && connectionErr.Category == "unavailable"
}