Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

46.2 KB
Raw Blame
package monitoring

import (
	"context"
	"errors"
	"fmt"
	"net"
	"net/http"
	"net/url"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/config"
	internalerrors "github.com/rcourtman/pulse-go-rewrite/internal/monitoring/errors"
	recoverymanager "github.com/rcourtman/pulse-go-rewrite/internal/recovery/manager"
	truenasmapper "github.com/rcourtman/pulse-go-rewrite/internal/recovery/mapper/truenas"
	"github.com/rcourtman/pulse-go-rewrite/internal/truenas"
	"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
	"github.com/rs/zerolog/log"
)

const defaultTrueNASPollInterval = 60 * time.Second

const defaultTrueNASHistoryReadTimeout = 10 * time.Second

// TrueNASConnectionPollError captures the last runtime poll error for one
// configured TrueNAS connection.
type TrueNASConnectionPollError struct {
	At       *time.Time `json:"at,omitempty"`
	Message  string     `json:"message,omitempty"`
	Category string     `json:"category,omitempty"`
}

// TrueNASConnectionPollStatus summarizes the runtime polling state for one
// configured TrueNAS connection.
type TrueNASConnectionPollStatus struct {
	IntervalSeconds     int                         `json:"intervalSeconds"`
	LastAttemptAt       *time.Time                  `json:"lastAttemptAt,omitempty"`
	LastSuccessAt       *time.Time                  `json:"lastSuccessAt,omitempty"`
	ConsecutiveFailures int                         `json:"consecutiveFailures,omitempty"`
	LastError           *TrueNASConnectionPollError `json:"lastError,omitempty"`
}

// TrueNASConnectionObservedSummary summarizes the canonical resources the most
// recent successful poll contributed for one configured TrueNAS connection.
type TrueNASConnectionObservedSummary struct {
	Host              string     `json:"host,omitempty"`
	ResourceID        string     `json:"resourceId,omitempty"`
	CollectedAt       *time.Time `json:"collectedAt,omitempty"`
	Systems           int        `json:"systems"`
	StoragePools      int        `json:"storagePools"`
	Datasets          int        `json:"datasets"`
	Apps              int        `json:"apps"`
	VMs               int        `json:"vms"`
	Shares            int        `json:"shares"`
	Disks             int        `json:"disks"`
	RecoveryArtifacts int        `json:"recoveryArtifacts"`
	// Telemetry is separate from poll success: inventory can be current even
	// when optional CPU/memory/IO collection failed or only partly succeeded.
	Telemetry *truenas.SystemTelemetryAvailability `json:"telemetry,omitempty"`
}

// TrueNASConnectionSummary merges poll health with the most recent discovered
// platform contribution for one configured TrueNAS connection.
type TrueNASConnectionSummary struct {
	Poll      *TrueNASConnectionPollStatus      `json:"poll,omitempty"`
	Observed  *TrueNASConnectionObservedSummary `json:"observed,omitempty"`
	Transport *truenas.TransportStatus          `json:"transport,omitempty"`
}

type trueNASConnectionRuntimeStatus struct {
	lastAttemptAt       time.Time
	lastSuccessAt       time.Time
	lastError           *TrueNASConnectionPollError
	consecutiveFailures int
	nextPollAt          time.Time
	observed            *TrueNASConnectionObservedSummary
}

// TrueNASPoller manages periodic polling of configured TrueNAS connections.
type TrueNASPoller struct {
	multiTenant     *config.MultiTenantPersistence
	recoveryManager *recoverymanager.Manager
	mu              sync.Mutex
	// providersByOrg is keyed by orgID, then connection ID.
	providersByOrg map[string]map[string]*truenas.Provider
	// configsByOrg is keyed by orgID, then connection ID.
	configsByOrg map[string]map[string]config.TrueNASInstance
	// statusByOrg is keyed by orgID, then connection ID.
	statusByOrg map[string]map[string]*trueNASConnectionRuntimeStatus
	// cachedRecordsByOrg is keyed by orgID, then connection ID.
	cachedRecordsByOrg map[string]map[string][]unifiedresources.IngestRecord
	cancel             context.CancelFunc
	stopped            chan struct{}
	interval           time.Duration
	explicitInterval   bool
}

type trueNASPollerProviderEntry struct {
	connectionID string
	provider     *truenas.Provider
}

type trueNASReplicationConnection struct {
	connectionID   string
	configuredHost string
	snapshot       *truenas.FixtureSnapshot
}

// NewTrueNASPoller builds a new TrueNAS poller with the provided poll interval.
func NewTrueNASPoller(multiTenant *config.MultiTenantPersistence, interval time.Duration, recoveryManager *recoverymanager.Manager) *TrueNASPoller {
	explicitInterval := interval > 0
	if interval <= 0 {
		interval = defaultTrueNASPollInterval
	}

	stopped := make(chan struct{})
	close(stopped)

	return &TrueNASPoller{
		multiTenant:        multiTenant,
		recoveryManager:    recoveryManager,
		providersByOrg:     make(map[string]map[string]*truenas.Provider),
		configsByOrg:       make(map[string]map[string]config.TrueNASInstance),
		statusByOrg:        make(map[string]map[string]*trueNASConnectionRuntimeStatus),
		cachedRecordsByOrg: make(map[string]map[string][]unifiedresources.IngestRecord),
		stopped:            stopped,
		interval:           interval,
		explicitInterval:   explicitInterval,
	}
}

// Start begins periodic TrueNAS polling if the feature flag is enabled.
func (p *TrueNASPoller) Start(ctx context.Context) {
	if p == nil || !truenas.IsFeatureEnabled() {
		return
	}
	startPollerLoop(ctx, &p.mu, &p.cancel, &p.stopped, p.syncConnections, p.pollAll, p.nextWaitDuration)
}

// Stop requests poller shutdown and waits up to five seconds for exit.
func (p *TrueNASPoller) Stop() {
	if p == nil {
		return
	}

	p.mu.Lock()
	cancel := p.cancel
	stopped := p.stopped
	p.mu.Unlock()

	if cancel != nil {
		cancel()
	}

	if stopped == nil {
		return
	}

	timer := time.NewTimer(5 * time.Second)
	defer timer.Stop()

	select {
	case <-stopped:
	case <-time.After(5 * time.Second):
		log.Warn().
			Str("component", "truenas_poller").
			Str("action", "stop").
			Dur("timeout", 5*time.Second).
			Msg("TrueNAS poller stop timed out waiting for shutdown")
	}
}

func (p *TrueNASPoller) syncConnections() {
	if p == nil {
		return
	}
	if p.multiTenant == nil {
		log.Warn().
			Str("component", "truenas_poller").
			Str("action", "sync_connections").
			Msg("TrueNAS poller cannot sync connections because multi-tenant persistence is nil")
		return
	}

	orgIDs, err := p.multiTenant.ListOrganizationIDs()
	if err != nil {
		log.Warn().
			Str("component", "truenas_poller").
			Str("action", "list_organizations").
			Err(err).
			Msg("TrueNAS poller failed to list organizations")
		return
	}

	// orgID -> connID -> instance
	configured := make(map[string]map[string]config.TrueNASInstance, len(orgIDs))
	active := make(map[string]map[string]config.TrueNASInstance, len(orgIDs))

	for _, orgID := range orgIDs {
		persistence, err := p.multiTenant.GetPersistence(orgID)
		if err != nil || persistence == nil {
			log.Warn().
				Str("component", "truenas_poller").
				Str("action", "get_persistence").
				Str("org_id", orgID).
				Err(err).
				Msg("TrueNAS poller failed to open tenant persistence")
			continue
		}

		instances, err := persistence.LoadTrueNASConfig()
		if err != nil {
			log.Warn().
				Str("component", "truenas_poller").
				Str("action", "load_truenas_config").
				Str("org_id", orgID).
				Err(err).
				Msg("TrueNAS poller failed to load TrueNAS config")
			continue
		}

		for i := range instances {
			instance := instances[i]
			id := strings.TrimSpace(instance.ID)
			if id == "" {
				continue
			}
			instance.ApplyDefaults()
			if configured[orgID] == nil {
				configured[orgID] = make(map[string]config.TrueNASInstance)
			}
			configured[orgID][id] = instance
			if !instance.Enabled {
				continue
			}
			if active[orgID] == nil {
				active[orgID] = make(map[string]config.TrueNASInstance)
			}
			active[orgID][id] = instance
		}
	}

	p.mu.Lock()
	defer p.mu.Unlock()

	// Ensure providers exist for active connections.
	for orgID, byConn := range active {
		if len(byConn) == 0 {
			continue
		}
		if p.providersByOrg[orgID] == nil {
			p.providersByOrg[orgID] = make(map[string]*truenas.Provider)
		}
		if p.configsByOrg[orgID] == nil {
			p.configsByOrg[orgID] = make(map[string]config.TrueNASInstance)
		}
		if p.statusByOrg[orgID] == nil {
			p.statusByOrg[orgID] = make(map[string]*trueNASConnectionRuntimeStatus)
		}
		if p.cachedRecordsByOrg[orgID] == nil {
			p.cachedRecordsByOrg[orgID] = make(map[string][]unifiedresources.IngestRecord)
		}

		for connID, instance := range byConn {
			previousConfig := p.configsByOrg[orgID][connID]
			existingProvider, exists := p.providersByOrg[orgID][connID]
			p.configsByOrg[orgID][connID] = instance
			p.alignConnectionNextPollLocked(orgID, connID, previousConfig, instance)
			if exists && !trueNASProviderConfigChanged(previousConfig, instance) {
				continue
			}

			client, err := truenas.NewClient(truenas.ClientConfig{
				Host:               instance.Host,
				Port:               instance.Port,
				APIKey:             instance.APIKey,
				Username:           instance.Username,
				Password:           instance.Password,
				UseHTTPS:           instance.UseHTTPS,
				InsecureSkipVerify: instance.InsecureSkipVerify,
				Fingerprint:        instance.Fingerprint,
			})
			if err != nil {
				now := time.Now().UTC()
				p.recordConnectionFailureLocked(orgID, connID, instance, err, now)
				log.Warn().
					Str("component", "truenas_poller").
					Str("action", "initialize_client").
					Str("org_id", orgID).
					Str("connection_id", connID).
					Err(err).
					Msg("TrueNAS poller failed to initialize client")
				continue
			}

			if existingProvider != nil {
				existingProvider.Close()
			}
			p.providersByOrg[orgID][connID] = truenas.NewLiveProviderForConnection(&truenas.APIFetcher{Client: client}, connID)
		}
	}

	// Prune providers for removed/disabled connections and removed orgs.
	for orgID, providers := range p.providersByOrg {
		activeByConn := active[orgID]
		for connID, provider := range providers {
			if activeByConn == nil {
				// Org no longer exists or has no active TrueNAS connections.
				if provider != nil {
					provider.Close()
				}
				delete(providers, connID)
				if p.configsByOrg[orgID] != nil {
					delete(p.configsByOrg[orgID], connID)
				}
				if p.cachedRecordsByOrg[orgID] != nil {
					delete(p.cachedRecordsByOrg[orgID], connID)
				}
				continue
			}
			if _, ok := activeByConn[connID]; !ok {
				if provider != nil {
					provider.Close()
				}
				delete(providers, connID)
				if p.configsByOrg[orgID] != nil {
					delete(p.configsByOrg[orgID], connID)
				}
				if p.cachedRecordsByOrg[orgID] != nil {
					delete(p.cachedRecordsByOrg[orgID], connID)
				}
			}
		}

		if len(providers) == 0 {
			delete(p.providersByOrg, orgID)
			delete(p.configsByOrg, orgID)
			delete(p.cachedRecordsByOrg, orgID)
		}
	}

	// Prune runtime status only when the connection is truly gone from config.
	for orgID, byConn := range p.statusByOrg {
		configuredByConn := configured[orgID]
		for connID := range byConn {
			if configuredByConn == nil {
				delete(byConn, connID)
				continue
			}
			if _, ok := configuredByConn[connID]; !ok {
				delete(byConn, connID)
			}
		}
		if len(byConn) == 0 {
			delete(p.statusByOrg, orgID)
		}
	}
}

// ConnectionSummaries returns per-connection runtime health and discovered
// contribution summaries for the supplied TrueNAS settings records.
func (p *TrueNASPoller) ConnectionSummaries(orgID string, instances []config.TrueNASInstance) map[string]TrueNASConnectionSummary {
	if len(instances) == 0 {
		return nil
	}

	orgID = normalizeTrueNASOrgID(orgID)
	summaries := make(map[string]TrueNASConnectionSummary, len(instances))

	for i := range instances {
		instance := instances[i]
		instance.ApplyDefaults()
		connID := strings.TrimSpace(instance.ID)
		if connID == "" {
			continue
		}

		summary := TrueNASConnectionSummary{
			Poll: &TrueNASConnectionPollStatus{
				IntervalSeconds: instance.EffectivePollIntervalSecs(),
			},
		}

		if p != nil {
			p.mu.Lock()
			status := p.cloneRuntimeStatusLocked(orgID, connID)
			var provider *truenas.Provider
			if byConnection := p.providersByOrg[orgID]; byConnection != nil {
				provider = byConnection[connID]
			}
			p.mu.Unlock()
			if status != nil {
				summary.Poll.LastAttemptAt = cloneTimePointer(status.LastAttemptAt)
				summary.Poll.LastSuccessAt = cloneTimePointer(status.LastSuccessAt)
				summary.Poll.ConsecutiveFailures = status.ConsecutiveFailures
				summary.Poll.LastError = cloneTrueNASConnectionPollError(status.LastError)
				summary.Observed = cloneTrueNASObservedSummary(status.Observed)
			}
			if provider != nil {
				transport := provider.TransportStatus()
				summary.Transport = &transport
			}
		}

		summaries[connID] = summary
	}

	return summaries
}

func (p *TrueNASPoller) nextWaitDuration(now time.Time) time.Duration {
	if p == nil {
		return defaultTrueNASPollInterval
	}

	p.mu.Lock()
	defer p.mu.Unlock()

	wait := p.interval
	for orgID, byConn := range p.configsByOrg {
		for connID, instance := range byConn {
			next := p.nextPollAtLocked(orgID, connID, instance)
			if next.IsZero() || !next.After(now) {
				return 0
			}
			until := next.Sub(now)
			if until < wait {
				wait = until
			}
		}
	}

	if wait < 0 {
		return 0
	}
	return wait
}

func (p *TrueNASPoller) pollAll(ctx context.Context) {
	if p == nil {
		return
	}
	p.mu.Lock()
	type providerEntry struct {
		orgID    string
		id       string
		config   config.TrueNASInstance
		provider *truenas.Provider
	}
	entries := make([]providerEntry, 0)
	now := time.Now().UTC()
	for orgID, providers := range p.providersByOrg {
		for id, provider := range providers {
			instance, ok := p.configsByOrg[orgID][id]
			if !ok {
				continue
			}
			if !p.connectionPollDueLocked(orgID, id, instance, now) {
				continue
			}
			entries = append(entries, providerEntry{
				orgID:    orgID,
				id:       id,
				config:   instance,
				provider: provider,
			})
		}
	}
	p.mu.Unlock()

	pm := getPollMetrics()
	refreshedOrgs := make(map[string]struct{})

	for _, entry := range entries {
		if entry.provider == nil {
			continue
		}

		start := time.Now().UTC()
		err := entry.provider.Refresh(ctx)
		end := time.Now().UTC()
		if err != nil {
			pm.RecordResult(PollResult{
				InstanceName: entry.id,
				InstanceType: "truenas",
				Success:      false,
				Error:        classifyTrueNASError(err, entry.id),
				StartTime:    start,
				EndTime:      end,
			})
			p.mu.Lock()
			p.recordConnectionFailureLocked(entry.orgID, entry.id, entry.config, err, end)
			p.mu.Unlock()
			log.Warn().
				Str("component", "truenas_poller").
				Str("action", "refresh_connection").
				Str("connection_id", entry.id).
				Err(err).
				Msg("TrueNAS poller refresh failed")
			continue
		}

		pm.RecordResult(PollResult{
			InstanceName: entry.id,
			InstanceType: "truenas",
			Success:      true,
			StartTime:    start,
			EndTime:      end,
		})

		snapshot := entry.provider.Snapshot()
		p.mu.Lock()
		p.recordConnectionSuccessLocked(entry.orgID, entry.id, entry.config, start, end, snapshot)
		p.mu.Unlock()
		refreshedOrgs[entry.orgID] = struct{}{}
		p.ingestRecoveryPoints(ctx, entry.orgID, entry.id, entry.provider)
	}

	for orgID := range refreshedOrgs {
		p.rebuildCachedRecordsForOrg(orgID)
	}
}

func (p *TrueNASPoller) rebuildCachedRecordsForOrg(orgID string) {
	if p == nil {
		return
	}

	p.mu.Lock()
	providers := p.providersByOrg[orgID]
	type providerSelection struct {
		connectionID   string
		configuredHost string
		provider       *truenas.Provider
	}
	selections := make([]providerSelection, 0, len(providers))
	providersByConnection := make(map[string]*truenas.Provider, len(providers))
	for connectionID, provider := range providers {
		if provider == nil {
			continue
		}
		instance := p.configsByOrg[orgID][connectionID]
		selections = append(selections, providerSelection{
			connectionID:   connectionID,
			configuredHost: instance.Host,
			provider:       provider,
		})
		providersByConnection[connectionID] = provider
	}
	p.mu.Unlock()

	connections := make([]trueNASReplicationConnection, 0, len(selections))
	for _, selection := range selections {
		connections = append(connections, trueNASReplicationConnection{
			connectionID:   selection.connectionID,
			configuredHost: selection.configuredHost,
			snapshot:       selection.provider.Snapshot(),
		})
	}

	targetsByConnection := expectedTrueNASReplicationReadOnlyTargets(connections)
	recordsByConnection := make(map[string][]unifiedresources.IngestRecord, len(connections))
	for _, connection := range connections {
		provider := providersByConnection[connection.connectionID]
		classified := applyTrueNASReplicationReadOnlyTargets(
			connection.snapshot,
			targetsByConnection[connection.connectionID],
		)
		recordsByConnection[connection.connectionID] = provider.RecordsFromSnapshot(classified)
	}

	p.mu.Lock()
	if p.cachedRecordsByOrg[orgID] == nil {
		p.cachedRecordsByOrg[orgID] = make(map[string][]unifiedresources.IngestRecord)
	}
	for connectionID, records := range recordsByConnection {
		// A concurrent settings update may have replaced or removed a
		// provider while snapshots were classified outside the lock.
		if p.providersByOrg[orgID][connectionID] != providersByConnection[connectionID] {
			continue
		}
		p.cachedRecordsByOrg[orgID][connectionID] = cloneIngestRecords(records)
	}
	p.mu.Unlock()
}

func expectedTrueNASReplicationReadOnlyTargets(connections []trueNASReplicationConnection) map[string][]string {
	aliases := make(map[string][]string)
	for _, connection := range connections {
		for _, alias := range dedupeTrueNASStrings([]string{
			normalizeTrueNASHost(connection.configuredHost),
			normalizeTrueNASHost(snapshotTrueNASHostname(connection.snapshot)),
		}) {
			aliases[alias] = append(aliases[alias], connection.connectionID)
		}
	}

	targetSets := make(map[string]map[string]struct{})
	addTarget := func(connectionID, dataset string) {
		connectionID = strings.TrimSpace(connectionID)
		dataset = strings.Trim(strings.TrimSpace(dataset), "/")
		if connectionID == "" || dataset == "" {
			return
		}
		if targetSets[connectionID] == nil {
			targetSets[connectionID] = make(map[string]struct{})
		}
		targetSets[connectionID][dataset] = struct{}{}
	}

	for _, connection := range connections {
		if connection.snapshot == nil {
			continue
		}
		for _, task := range connection.snapshot.ReplicationTasks {
			switch strings.ToUpper(strings.TrimSpace(task.ReadOnlyMode)) {
			case "SET", "REQUIRE":
			default:
				continue
			}

			direction := strings.ToUpper(strings.TrimSpace(task.Direction))
			transport := strings.ToUpper(strings.TrimSpace(task.Transport))
			switch {
			case direction == "PULL" || transport == "LOCAL":
				addTarget(connection.connectionID, task.TargetDataset)
			case direction == "PUSH":
				targetHost := normalizeTrueNASHost(task.TargetHost)
				candidates := dedupeTrueNASStrings(aliases[targetHost])
				if targetHost == "" || len(candidates) != 1 {
					continue
				}
				addTarget(candidates[0], task.TargetDataset)
			}
		}
	}

	targets := make(map[string][]string, len(targetSets))
	for connectionID, set := range targetSets {
		for dataset := range set {
			targets[connectionID] = append(targets[connectionID], dataset)
		}
		sort.Strings(targets[connectionID])
	}
	return targets
}

func applyTrueNASReplicationReadOnlyTargets(snapshot *truenas.FixtureSnapshot, targets []string) *truenas.FixtureSnapshot {
	if snapshot == nil {
		return nil
	}
	classified := *snapshot
	classified.Datasets = append([]truenas.Dataset(nil), snapshot.Datasets...)
	for i := range classified.Datasets {
		if !classified.Datasets[i].ReadOnly {
			continue
		}
		dataset := strings.Trim(strings.TrimSpace(classified.Datasets[i].Name), "/")
		for _, target := range targets {
			target = strings.Trim(strings.TrimSpace(target), "/")
			if target != "" && (dataset == target || strings.HasPrefix(dataset, target+"/")) {
				classified.Datasets[i].ReadOnlyReason = truenas.DatasetReadOnlyReplicationTarget
				break
			}
		}
	}
	return &classified
}

func snapshotTrueNASHostname(snapshot *truenas.FixtureSnapshot) string {
	if snapshot == nil {
		return ""
	}
	return snapshot.System.Hostname
}

func normalizeTrueNASHost(raw string) string {
	raw = strings.TrimSpace(raw)
	if raw == "" {
		return ""
	}
	candidate := raw
	if !strings.Contains(candidate, "://") {
		candidate = "//" + candidate
	}
	if parsed, err := url.Parse(candidate); err == nil {
		if hostname := parsed.Hostname(); hostname != "" {
			return strings.TrimSuffix(strings.ToLower(hostname), ".")
		}
	}
	if host, _, err := net.SplitHostPort(raw); err == nil {
		raw = host
	}
	return strings.TrimSuffix(strings.ToLower(strings.Trim(raw, "[]")), ".")
}

func dedupeTrueNASStrings(values []string) []string {
	if len(values) == 0 {
		return nil
	}
	seen := make(map[string]struct{}, len(values))
	out := make([]string, 0, len(values))
	for _, value := range values {
		value = strings.TrimSpace(value)
		if value == "" {
			continue
		}
		if _, exists := seen[value]; exists {
			continue
		}
		seen[value] = struct{}{}
		out = append(out, value)
	}
	return out
}

type trueNASConnectionRuntimeSnapshot struct {
	LastAttemptAt       *time.Time
	LastSuccessAt       *time.Time
	ConsecutiveFailures int
	LastError           *TrueNASConnectionPollError
	Observed            *TrueNASConnectionObservedSummary
}

func (p *TrueNASPoller) cloneRuntimeStatusLocked(orgID string, connID string) *trueNASConnectionRuntimeSnapshot {
	if p == nil {
		return nil
	}
	byConn := p.statusByOrg[orgID]
	if byConn == nil {
		return nil
	}
	status := byConn[connID]
	if status == nil {
		return nil
	}
	return &trueNASConnectionRuntimeSnapshot{
		LastAttemptAt:       timePointerIfSet(status.lastAttemptAt),
		LastSuccessAt:       timePointerIfSet(status.lastSuccessAt),
		ConsecutiveFailures: status.consecutiveFailures,
		LastError:           cloneTrueNASConnectionPollError(status.lastError),
		Observed:            cloneTrueNASObservedSummary(status.observed),
	}
}

func (p *TrueNASPoller) ensureConnectionRuntimeStatusLocked(orgID, connID string) *trueNASConnectionRuntimeStatus {
	if p.statusByOrg[orgID] == nil {
		p.statusByOrg[orgID] = make(map[string]*trueNASConnectionRuntimeStatus)
	}
	if p.statusByOrg[orgID][connID] == nil {
		p.statusByOrg[orgID][connID] = &trueNASConnectionRuntimeStatus{}
	}
	return p.statusByOrg[orgID][connID]
}

func (p *TrueNASPoller) alignConnectionNextPollLocked(
	orgID string,
	connID string,
	previous config.TrueNASInstance,
	next config.TrueNASInstance,
) {
	status := p.ensureConnectionRuntimeStatusLocked(orgID, connID)
	if status == nil || status.lastAttemptAt.IsZero() {
		return
	}
	previousInterval := p.effectiveRuntimePollInterval(previous)
	nextInterval := p.effectiveRuntimePollInterval(next)
	if previousInterval == nextInterval {
		return
	}
	status.nextPollAt = status.lastAttemptAt.Add(nextInterval)
}

func (p *TrueNASPoller) connectionPollDueLocked(
	orgID string,
	connID string,
	instance config.TrueNASInstance,
	now time.Time,
) bool {
	next := p.nextPollAtLocked(orgID, connID, instance)
	return next.IsZero() || !next.After(now)
}

func (p *TrueNASPoller) nextPollAtLocked(orgID string, connID string, instance config.TrueNASInstance) time.Time {
	status := p.ensureConnectionRuntimeStatusLocked(orgID, connID)
	if status == nil {
		return time.Time{}
	}
	if status.nextPollAt.IsZero() && !status.lastAttemptAt.IsZero() {
		status.nextPollAt = status.lastAttemptAt.Add(p.effectiveRuntimePollInterval(instance))
	}
	return status.nextPollAt
}

func (p *TrueNASPoller) effectiveRuntimePollInterval(instance config.TrueNASInstance) time.Duration {
	base := time.Duration(instance.EffectivePollIntervalSecs()) * time.Second
	if p != nil && p.explicitInterval && p.interval > 0 && p.interval < base {
		return p.interval
	}
	return base
}

func (p *TrueNASPoller) recordConnectionSuccessLocked(
	orgID string,
	connID string,
	instance config.TrueNASInstance,
	startedAt time.Time,
	at time.Time,
	snapshot *truenas.FixtureSnapshot,
) {
	status := p.ensureConnectionRuntimeStatusLocked(orgID, connID)
	status.lastAttemptAt = at
	status.lastSuccessAt = at
	status.lastError = nil
	status.consecutiveFailures = 0
	// Successful cycles target start-to-start cadence, but never immediately
	// hammer a slow appliance with another refresh. Keep at least five seconds
	// idle (or the entire interval for short-interval configurations). Failures
	// continue to use a full completion-based retry interval below.
	interval := p.effectiveRuntimePollInterval(instance)
	idleGap := min(5*time.Second, interval)
	status.nextPollAt = startedAt.Add(interval)
	if earliest := at.Add(idleGap); status.nextPollAt.Before(earliest) {
		status.nextPollAt = earliest
	}
	if snapshot != nil {
		status.observed = buildTrueNASObservedSummary(snapshot)
	}
}

func (p *TrueNASPoller) recordConnectionFailureLocked(
	orgID string,
	connID string,
	instance config.TrueNASInstance,
	err error,
	at time.Time,
) {
	status := p.ensureConnectionRuntimeStatusLocked(orgID, connID)
	status.lastAttemptAt = at
	status.consecutiveFailures++
	status.nextPollAt = at.Add(p.effectiveRuntimePollInterval(instance))

	category := ""
	if monitorErr := classifyTrueNASError(err, connID); monitorErr != nil {
		category = string(monitorErr.Type)
	}
	status.lastError = &TrueNASConnectionPollError{
		At:       timePointerIfSet(at),
		Message:  strings.TrimSpace(err.Error()),
		Category: category,
	}
}

func buildTrueNASObservedSummary(snapshot *truenas.FixtureSnapshot) *TrueNASConnectionObservedSummary {
	if snapshot == nil {
		return nil
	}

	host := strings.TrimSpace(snapshot.System.Hostname)
	resourceID := host
	collectedAt := snapshot.CollectedAt
	if collectedAt.IsZero() {
		collectedAt = snapshot.System.CollectedAt
	}

	summary := &TrueNASConnectionObservedSummary{
		Host:              host,
		ResourceID:        resourceID,
		Systems:           0,
		StoragePools:      len(snapshot.Pools),
		Datasets:          len(snapshot.Datasets),
		Apps:              len(snapshot.Apps),
		VMs:               len(snapshot.VMs),
		Shares:            len(snapshot.Shares),
		Disks:             len(snapshot.Disks),
		RecoveryArtifacts: len(snapshot.ZFSSnapshots) + len(snapshot.ReplicationTasks),
		Telemetry:         cloneTrueNASTelemetryAvailability(snapshot.System.Telemetry),
	}
	if host != "" || resourceID != "" {
		summary.Systems = 1
	}
	summary.CollectedAt = timePointerIfSet(collectedAt)
	return summary
}

func normalizeTrueNASOrgID(orgID string) string {
	orgID = strings.TrimSpace(orgID)
	if orgID == "" {
		return "default"
	}
	return orgID
}

func timePointerIfSet(value time.Time) *time.Time {
	if value.IsZero() {
		return nil
	}
	copied := value
	return &copied
}

func cloneTimePointer(value *time.Time) *time.Time {
	if value == nil || value.IsZero() {
		return nil
	}
	copied := *value
	return &copied
}

func cloneTrueNASConnectionPollError(value *TrueNASConnectionPollError) *TrueNASConnectionPollError {
	if value == nil {
		return nil
	}
	return &TrueNASConnectionPollError{
		At:       cloneTimePointer(value.At),
		Message:  strings.TrimSpace(value.Message),
		Category: strings.TrimSpace(value.Category),
	}
}

func cloneTrueNASObservedSummary(value *TrueNASConnectionObservedSummary) *TrueNASConnectionObservedSummary {
	if value == nil {
		return nil
	}
	return &TrueNASConnectionObservedSummary{
		Host:              strings.TrimSpace(value.Host),
		ResourceID:        strings.TrimSpace(value.ResourceID),
		CollectedAt:       cloneTimePointer(value.CollectedAt),
		Systems:           value.Systems,
		StoragePools:      value.StoragePools,
		Datasets:          value.Datasets,
		Apps:              value.Apps,
		VMs:               value.VMs,
		Shares:            value.Shares,
		Disks:             value.Disks,
		RecoveryArtifacts: value.RecoveryArtifacts,
		Telemetry:         cloneTrueNASTelemetryAvailability(value.Telemetry),
	}
}

func cloneTrueNASTelemetryAvailability(value *truenas.SystemTelemetryAvailability) *truenas.SystemTelemetryAvailability {
	if value == nil {
		return nil
	}
	cloned := *value
	return &cloned
}

func (p *TrueNASPoller) ingestRecoveryPoints(ctx context.Context, orgID string, connectionID string, provider *truenas.Provider) {
	if p == nil || p.recoveryManager == nil || provider == nil {
		return
	}

	store, err := p.recoveryManager.StoreForOrg(orgID)
	if err != nil {
		log.Warn().
			Str("component", "truenas_poller").
			Str("action", "recovery_store").
			Err(err).
			Msg("TrueNAS poller failed to open recovery store")
		return
	}

	snapshot := provider.Snapshot()
	points := truenasmapper.FromTrueNASSnapshot(connectionID, snapshot)
	if len(points) == 0 {
		return
	}

	// Best-effort cap in case a system has an enormous snapshot history.
	const maxPointsPerPoll = 2000
	if len(points) > maxPointsPerPoll {
		points = points[:maxPointsPerPoll]
	}

	if err := store.UpsertPoints(ctx, points); err != nil {
		log.Warn().
			Str("component", "truenas_poller").
			Str("action", "ingest_recovery_points").
			Str("connection_id", strings.TrimSpace(connectionID)).
			Err(err).
			Msg("TrueNAS poller failed to ingest recovery points")
	}
}

// GetCurrentRecords returns the latest known TrueNAS records across active connections.
func (p *TrueNASPoller) GetCurrentRecords() []unifiedresources.IngestRecord {
	return p.GetCurrentRecordsForOrg("default")
}

// SupplementalRecords adapts TrueNAS records to the monitor supplemental
// provider contract.
// SupplementalRecordsMockMode binds native records to real infrastructure.
// Mock-mode publications must not read this cached inventory before the
// router's post-switch hook replaces the platform provider pair.
func (*TrueNASPoller) SupplementalRecordsMockMode() bool { return false }

func (p *TrueNASPoller) SupplementalRecords(_ *Monitor, orgID string) []unifiedresources.IngestRecord {
	return p.GetCurrentRecordsForOrg(orgID)
}

// PhysicalDiskTemperatureHistory exposes native TrueNAS disk temperature
// history through the canonical monitoring chart boundary.
func (p *TrueNASPoller) PhysicalDiskTemperatureHistory(_ *Monitor, orgID string, duration time.Duration) map[string][]MetricPoint {
	if p == nil || !truenas.IsFeatureEnabled() {
		return nil
	}

	orgID = strings.TrimSpace(orgID)
	if orgID == "" {
		orgID = "default"
	}

	entries := p.providerEntriesForOrg(orgID)
	if len(entries) == 0 {
		return nil
	}

	ctx, cancel := context.WithTimeout(context.Background(), defaultTrueNASHistoryReadTimeout)
	defer cancel()

	history := make(map[string][]MetricPoint)
	owners := make(map[string]trueNASPollerProviderEntry)
	ambiguous := make(map[string]bool)
	for _, entry := range entries {
		nativeHistory, err := entry.provider.PhysicalDiskTemperatureHistory(ctx, duration)
		if err != nil {
			log.Warn().
				Str("component", "truenas_poller").
				Str("action", "disk_temperature_history").
				Str("org_id", orgID).
				Str("connection_id", strings.TrimSpace(entry.connectionID)).
				Err(err).
				Msg("TrueNAS poller failed to read native disk temperature history")
		}
		for resourceID, points := range nativeHistory {
			if strings.TrimSpace(resourceID) == "" || len(points) == 0 || ambiguous[resourceID] {
				continue
			}
			if owner, exists := owners[resourceID]; exists && owner.connectionID != entry.connectionID {
				delete(history, resourceID)
				delete(owners, resourceID)
				ambiguous[resourceID] = true
				continue
			}
			converted := make([]MetricPoint, len(points))
			for i, point := range points {
				converted[i] = MetricPoint{
					Timestamp: point.Timestamp,
					Value:     point.Value,
				}
			}
			history[resourceID] = converted
			owners[resourceID] = entry
		}
	}
	// Do not serve a provider that was removed or replaced while any of the
	// native reads were in flight. No poller lock is held during network I/O.
	p.mu.Lock()
	for resourceID, owner := range owners {
		if !truenas.IsFeatureEnabled() || p.providersByOrg[orgID][owner.connectionID] != owner.provider {
			delete(history, resourceID)
		}
	}
	p.mu.Unlock()
	if len(history) == 0 {
		return nil
	}
	return history
}

// GuestMetricHistory exposes native TrueNAS host history through the canonical
// guest-chart boundary when local Pulse history is shallow.
func (p *TrueNASPoller) GuestMetricHistory(_ *Monitor, orgID string, resourceType string, duration time.Duration) map[string]map[string][]MetricPoint {
	return p.guestMetricHistory(orgID, resourceType, "", duration)
}

// GuestMetricHistoryForResource selects the exact tenant-local host before any
// native read. A missing host must not turn a drawer request into a fleet scan.
func (p *TrueNASPoller) GuestMetricHistoryForResource(_ *Monitor, orgID, resourceType, resourceID string, duration time.Duration) map[string][]MetricPoint {
	resourceID = strings.TrimSpace(resourceID)
	if resourceID == "" {
		return nil
	}
	return p.guestMetricHistory(orgID, resourceType, resourceID, duration)[resourceID]
}

func (p *TrueNASPoller) guestMetricHistory(orgID, resourceType, selectedID string, duration time.Duration) map[string]map[string][]MetricPoint {
	if p == nil || !truenas.IsFeatureEnabled() || strings.TrimSpace(resourceType) != "agent" {
		return nil
	}

	orgID = strings.TrimSpace(orgID)
	if orgID == "" {
		orgID = "default"
	}

	entries := p.providerEntriesForOrg(orgID)
	if len(entries) == 0 {
		return nil
	}

	ctx, cancel := context.WithTimeout(context.Background(), defaultTrueNASHistoryReadTimeout)
	defer cancel()

	history := make(map[string]map[string][]MetricPoint)
	owners := make(map[string]trueNASPollerProviderEntry)
	for _, entry := range entries {
		if selectedID != "" && entry.provider.SystemMetricResourceID() != selectedID {
			continue
		}
		resourceID, nativeHistory, err := entry.provider.SystemMetricHistory(ctx, duration)
		if err != nil {
			log.Warn().
				Str("component", "truenas_poller").
				Str("action", "guest_metric_history").
				Str("resource_type", resourceType).
				Str("org_id", orgID).
				Str("connection_id", strings.TrimSpace(entry.connectionID)).
				Err(err).
				Msg("TrueNAS poller failed to read native guest history")
			continue
		}
		resourceID = strings.TrimSpace(resourceID)
		if resourceID == "" || len(nativeHistory) == 0 || (selectedID != "" && resourceID != selectedID) {
			continue
		}
		// Do not return a provider removed or replaced while its read was in flight.
		p.mu.Lock()
		current := truenas.IsFeatureEnabled() && p.providersByOrg[orgID][entry.connectionID] == entry.provider
		p.mu.Unlock()
		if !current {
			continue
		}

		converted := make(map[string][]MetricPoint, len(nativeHistory))
		for metricType, points := range nativeHistory {
			if strings.TrimSpace(metricType) == "" || len(points) == 0 {
				continue
			}
			series := make([]MetricPoint, len(points))
			for i, point := range points {
				series[i] = MetricPoint{
					Timestamp: point.Timestamp,
					Value:     point.Value,
				}
			}
			converted[metricType] = series
		}
		if len(converted) == 0 {
			continue
		}
		history[resourceID] = converted
		owners[resourceID] = entry
	}
	// An earlier provider can lose ownership while a later native read is
	// outstanding. Fence the complete result, not just each individual read.
	p.mu.Lock()
	for resourceID, owner := range owners {
		if !truenas.IsFeatureEnabled() || p.providersByOrg[orgID][owner.connectionID] != owner.provider {
			delete(history, resourceID)
		}
	}
	p.mu.Unlock()
	if len(history) == 0 {
		return nil
	}
	return history
}

// SnapshotOwnedSources declares source-native ingest ownership for legacy
// snapshot suppression (default/global call path).
func (p *TrueNASPoller) SnapshotOwnedSources() []unifiedresources.DataSource {
	return []unifiedresources.DataSource{unifiedresources.SourceTrueNAS}
}

// SnapshotOwnedSourcesForOrg declares source-native ingest ownership for a
// specific tenant.
func (p *TrueNASPoller) SnapshotOwnedSourcesForOrg(string) []unifiedresources.DataSource {
	return []unifiedresources.DataSource{unifiedresources.SourceTrueNAS}
}

// SupplementalInventoryReadyAt reports when the current org-scoped TrueNAS
// inventory has reached a settled initial baseline that has to be reflected in
// the canonical monitor store before support surfaces consume monitored-system
// counts.
func (p *TrueNASPoller) SupplementalInventoryReadyAt(_ *Monitor, orgID string) (time.Time, bool) {
	if p == nil {
		return time.Time{}, true
	}

	orgID = normalizeTrueNASOrgID(orgID)
	configs := p.activeConnectionConfigsForOrg(orgID)
	if len(configs) == 0 {
		return time.Time{}, true
	}

	p.mu.Lock()
	defer p.mu.Unlock()

	var watermark time.Time
	statusByConn := p.statusByOrg[orgID]
	for connID := range configs {
		status := statusByConn[connID]
		if status == nil || status.lastAttemptAt.IsZero() {
			return time.Time{}, false
		}
		if status.lastAttemptAt.After(watermark) {
			watermark = status.lastAttemptAt
		}
	}

	return watermark, true
}

// GetCurrentRecordsForOrg returns the latest known TrueNAS records for the specified org.
func (p *TrueNASPoller) GetCurrentRecordsForOrg(orgID string) []unifiedresources.IngestRecord {
	if p == nil {
		return nil
	}

	orgID = strings.TrimSpace(orgID)
	if orgID == "" {
		orgID = "default"
	}

	p.mu.Lock()
	defer p.mu.Unlock()

	recordsByConn := p.cachedRecordsByOrg[orgID]
	if len(recordsByConn) == 0 {
		return nil
	}

	connectionIDs := make([]string, 0, len(recordsByConn))
	for id := range recordsByConn {
		connectionIDs = append(connectionIDs, id)
	}
	sort.Strings(connectionIDs)

	total := 0
	for _, id := range connectionIDs {
		total += len(recordsByConn[id])
	}
	if total == 0 {
		return nil
	}

	records := make([]unifiedresources.IngestRecord, 0, total)
	for _, id := range connectionIDs {
		records = append(records, cloneIngestRecords(recordsByConn[id])...)
	}
	return records
}

func (p *TrueNASPoller) activeConnectionConfigsForOrg(orgID string) map[string]config.TrueNASInstance {
	orgID = normalizeTrueNASOrgID(orgID)
	if p == nil {
		return nil
	}

	p.mu.Lock()
	cached := cloneTrueNASConnectionConfigMap(p.configsByOrg[orgID])
	p.mu.Unlock()
	return loadActiveInstanceConfigs(
		cached,
		p.multiTenant,
		orgID,
		(*config.ConfigPersistence).LoadTrueNASConfig,
		func(instance *config.TrueNASInstance) { instance.ApplyDefaults() },
		func(instance config.TrueNASInstance) bool { return instance.Enabled },
		func(instance config.TrueNASInstance) string { return instance.ID },
	)
}

func cloneTrueNASConnectionConfigMap(
	in map[string]config.TrueNASInstance,
) map[string]config.TrueNASInstance {
	if len(in) == 0 {
		return nil
	}
	out := make(map[string]config.TrueNASInstance, len(in))
	for connID, instance := range in {
		out[connID] = instance
	}
	return out
}

// ControlApp executes a canonical TrueNAS app action for one tenant-scoped
// API-backed app-container resource and refreshes the cached records afterward.
func (p *TrueNASPoller) ControlApp(ctx context.Context, orgID, host, appID, action string) (*truenas.App, error) {
	if p == nil {
		return nil, fmt.Errorf("truenas poller is nil")
	}
	if !truenas.IsFeatureEnabled() {
		return nil, fmt.Errorf("truenas integration is disabled")
	}

	orgID = strings.TrimSpace(orgID)
	if orgID == "" {
		orgID = "default"
	}
	host = strings.TrimSpace(host)
	appID = strings.TrimSpace(appID)
	action = strings.ToLower(strings.TrimSpace(action))
	if appID == "" {
		return nil, fmt.Errorf("truenas app id is required")
	}

	entry, currentApp, err := p.findProviderEntryForApp(orgID, host, appID)
	if err != nil {
		return nil, err
	}

	var nextSnapshot *truenas.FixtureSnapshot
	switch action {
	case "start", "stop":
		nextSnapshot, err = entry.provider.ControlApp(ctx, trueNASAppCanonicalID(*currentApp), action)
	case "restart":
		if trueNASAppRunning(currentApp) {
			if _, err = entry.provider.ControlApp(ctx, trueNASAppCanonicalID(*currentApp), "stop"); err != nil {
				return nil, err
			}
		}
		nextSnapshot, err = entry.provider.ControlApp(ctx, trueNASAppCanonicalID(*currentApp), "start")
	default:
		return nil, fmt.Errorf("unsupported truenas app action %q", action)
	}
	if err != nil {
		return nil, err
	}

	p.rebuildCachedRecordsForOrg(orgID)
	p.ingestRecoveryPoints(ctx, orgID, entry.connectionID, entry.provider)

	if updatedApp, ok := findTrueNASAppSnapshot(nextSnapshot, host, appID); ok {
		appCopy := *updatedApp
		return &appCopy, nil
	}
	appCopy := *currentApp
	return &appCopy, nil
}

// ReadAppLogs executes a canonical bounded log read for one tenant-scoped
// TrueNAS-managed app-container resource.
func (p *TrueNASPoller) ReadAppLogs(ctx context.Context, orgID, host, appID, containerRef string, tailLines int) (*truenas.AppLogResult, error) {
	if p == nil {
		return nil, fmt.Errorf("truenas poller is nil")
	}
	if !truenas.IsFeatureEnabled() {
		return nil, fmt.Errorf("truenas integration is disabled")
	}

	orgID = strings.TrimSpace(orgID)
	if orgID == "" {
		orgID = "default"
	}
	host = strings.TrimSpace(host)
	appID = strings.TrimSpace(appID)
	if appID == "" {
		return nil, fmt.Errorf("truenas app id is required")
	}

	entry, _, err := p.findProviderEntryForApp(orgID, host, appID)
	if err != nil {
		return nil, err
	}
	return entry.provider.ReadAppLogs(ctx, appID, containerRef, tailLines)
}

// GetAppConfig reads the current TrueNAS app configuration/runtime shape for
// one tenant-scoped API-backed app-container resource.
func (p *TrueNASPoller) GetAppConfig(ctx context.Context, orgID, host, appID string) (*truenas.AppConfigResult, error) {
	if p == nil {
		return nil, fmt.Errorf("truenas poller is nil")
	}
	if !truenas.IsFeatureEnabled() {
		return nil, fmt.Errorf("truenas integration is disabled")
	}

	orgID = strings.TrimSpace(orgID)
	if orgID == "" {
		orgID = "default"
	}
	host = strings.TrimSpace(host)
	appID = strings.TrimSpace(appID)
	if appID == "" {
		return nil, fmt.Errorf("truenas app id is required")
	}

	entry, currentApp, err := p.findProviderEntryForApp(orgID, host, appID)
	if err != nil {
		return nil, err
	}

	result, err := entry.provider.GetAppConfig(ctx, trueNASAppCanonicalID(*currentApp))
	if err != nil {
		return nil, err
	}
	if result == nil {
		return nil, fmt.Errorf("truenas app %q configuration is unavailable", appID)
	}
	return result, nil
}

func cloneIngestRecords(records []unifiedresources.IngestRecord) []unifiedresources.IngestRecord {
	if len(records) == 0 {
		return nil
	}
	cloned := make([]unifiedresources.IngestRecord, len(records))
	copy(cloned, records)
	return cloned
}

func (p *TrueNASPoller) providerEntriesForOrg(orgID string) []trueNASPollerProviderEntry {
	if p == nil {
		return nil
	}

	p.mu.Lock()
	defer p.mu.Unlock()

	entries := make([]trueNASPollerProviderEntry, 0, len(p.providersByOrg[orgID]))
	for connectionID, provider := range p.providersByOrg[orgID] {
		if provider == nil {
			continue
		}
		entries = append(entries, trueNASPollerProviderEntry{
			connectionID: connectionID,
			provider:     provider,
		})
	}
	return entries
}

func (p *TrueNASPoller) findProviderEntryForApp(orgID, host, appID string) (trueNASPollerProviderEntry, *truenas.App, error) {
	for _, entry := range p.providerEntriesForOrg(orgID) {
		currentSnapshot := entry.provider.Snapshot()
		currentApp, ok := findTrueNASAppSnapshot(currentSnapshot, host, appID)
		if ok {
			return entry, currentApp, nil
		}
	}
	return trueNASPollerProviderEntry{}, nil, fmt.Errorf("truenas app %q was not found for org %q", appID, orgID)
}

func findTrueNASAppSnapshot(snapshot *truenas.FixtureSnapshot, host, appID string) (*truenas.App, bool) {
	if snapshot == nil {
		return nil, false
	}
	if host != "" && !strings.EqualFold(strings.TrimSpace(snapshot.System.Hostname), host) {
		return nil, false
	}
	for i := range snapshot.Apps {
		app := &snapshot.Apps[i]
		if strings.EqualFold(trueNASAppCanonicalID(*app), appID) || strings.EqualFold(strings.TrimSpace(app.Name), appID) {
			return app, true
		}
	}
	return nil, false
}

func trueNASAppCanonicalID(app truenas.App) string {
	if id := strings.TrimSpace(app.ID); id != "" {
		return id
	}
	return strings.TrimSpace(app.Name)
}

func trueNASAppRunning(app *truenas.App) bool {
	if app == nil {
		return false
	}
	if strings.EqualFold(strings.TrimSpace(app.State), "RUNNING") {
		return true
	}
	for _, container := range app.Containers {
		if strings.EqualFold(strings.TrimSpace(container.State), "running") {
			return true
		}
	}
	return false
}

func trueNASProviderConfigChanged(previous, next config.TrueNASInstance) bool {
	return strings.TrimSpace(previous.Host) != strings.TrimSpace(next.Host) ||
		previous.Port != next.Port ||
		strings.TrimSpace(previous.APIKey) != strings.TrimSpace(next.APIKey) ||
		strings.TrimSpace(previous.Username) != strings.TrimSpace(next.Username) ||
		strings.TrimSpace(previous.Password) != strings.TrimSpace(next.Password) ||
		previous.UseHTTPS != next.UseHTTPS ||
		previous.InsecureSkipVerify != next.InsecureSkipVerify ||
		strings.TrimSpace(previous.Fingerprint) != strings.TrimSpace(next.Fingerprint)
}

// classifyTrueNASError wraps a TrueNAS API error in MonitorError for metrics classification.
func classifyTrueNASError(err error, connectionID string) *internalerrors.MonitorError {
	if err == nil {
		return nil
	}

	errType := internalerrors.ErrorTypeAPI
	retryable := true

	var apiErr *truenas.APIError
	if errors.As(err, &apiErr) {
		switch {
		case apiErr.StatusCode == http.StatusUnauthorized || apiErr.StatusCode == http.StatusForbidden:
			errType = internalerrors.ErrorTypeAuth
			retryable = false
		case apiErr.StatusCode == http.StatusRequestTimeout || apiErr.StatusCode == http.StatusGatewayTimeout:
			errType = internalerrors.ErrorTypeTimeout
		default:
			errType = internalerrors.ErrorTypeAPI
		}
	} else {
		var authErr *truenas.RPCAuthError
		if errors.As(err, &authErr) {
			errType = internalerrors.ErrorTypeAuth
			retryable = false
		}
		var handshakeErr *truenas.RPCHandshakeError
		if errors.As(err, &handshakeErr) &&
			(handshakeErr.StatusCode == http.StatusUnauthorized || handshakeErr.StatusCode == http.StatusForbidden) {
			errType = internalerrors.ErrorTypeAuth
			retryable = false
		}
		var rpcErr *truenas.RPCError
		if errors.As(err, &rpcErr) {
			message := strings.ToLower(rpcErr.Message + " " + rpcErr.Reason)
			if strings.HasPrefix(rpcErr.Method, "auth.") ||
				strings.EqualFold(rpcErr.Errname, "EACCES") ||
				strings.EqualFold(rpcErr.Errname, "EPERM") ||
				strings.Contains(message, "not authorized") ||
				strings.Contains(message, "permission") {
				errType = internalerrors.ErrorTypeAuth
				retryable = false
			}
		}
		// Transport-level errors: timeout takes precedence over generic connection failures.
		var netErr net.Error
		if (errors.As(err, &netErr) && netErr.Timeout()) || errors.Is(err, context.DeadlineExceeded) {
			errType = internalerrors.ErrorTypeTimeout
		} else {
			var netOpErr *net.OpError
			if errors.As(err, &netOpErr) {
				errType = internalerrors.ErrorTypeConnection
			}
		}
	}

	return &internalerrors.MonitorError{
		Type:      errType,
		Op:        "truenas_poll",
		Instance:  connectionID,
		Err:       err,
		Timestamp: time.Now(),
		Retryable: retryable,
	}
}