Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

44.8 KB
Raw Blame
package monitoring

import (
	"context"
	"fmt"
	"math"
	"net"
	"sort"
	"strings"
	"time"

	"github.com/google/uuid"
	"github.com/rcourtman/pulse-go-rewrite/internal/availabilityprobe"
	"github.com/rcourtman/pulse-go-rewrite/internal/config"
	"github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust"
	"github.com/rcourtman/pulse-go-rewrite/internal/storagehealth"
	"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
	pkgmetrics "github.com/rcourtman/pulse-go-rewrite/pkg/metrics"
	"github.com/rcourtman/pulse-go-rewrite/pkg/tlsutil"
	"github.com/rs/zerolog/log"
)

type tlsCert = tlsutil.CertificateObservation

// AvailabilityProbeStatus captures the last observed state of an agentless
// endpoint probe.
type AvailabilityProbeStatus struct {
	TargetID               string                                  `json:"targetId"`
	Name                   string                                  `json:"name"`
	TargetKind             string                                  `json:"targetKind,omitempty"`
	Address                string                                  `json:"address"`
	Protocol               string                                  `json:"protocol"`
	Outcome                string                                  `json:"outcome,omitempty"`
	TransportOutcome       string                                  `json:"transportOutcome,omitempty"`
	ApplicationOutcome     string                                  `json:"applicationOutcome,omitempty"`
	ApplicationStatusCode  int                                     `json:"applicationStatusCode,omitempty"`
	ApplicationFailureCode string                                  `json:"applicationFailureCode,omitempty"`
	Enabled                bool                                    `json:"enabled"`
	Available              bool                                    `json:"available"`
	LastChecked            time.Time                               `json:"lastChecked,omitempty"`
	LastSuccess            time.Time                               `json:"lastSuccess,omitempty"`
	LatencyMillis          int64                                   `json:"latencyMillis,omitempty"`
	ConsecutiveFailures    int                                     `json:"consecutiveFailures,omitempty"`
	LastError              string                                  `json:"lastError,omitempty"`
	FailureThreshold       int                                     `json:"failureThreshold,omitempty"`
	ProbeAgentID           string                                  `json:"probeAgentId,omitempty"`
	AggregateState         string                                  `json:"aggregateState,omitempty"`
	Disagreement           bool                                    `json:"disagreement,omitempty"`
	ExpectedLocations      int                                     `json:"expectedLocations,omitempty"`
	ReportingLocations     int                                     `json:"reportingLocations,omitempty"`
	Locations              []AvailabilityObservationLocationStatus `json:"locations,omitempty"`
	Certificate            *tlsCert                                `json:"certificate,omitempty"`
	CertificateCurrent     bool                                    `json:"-"`
	// ProbeReportReceivedAt is server-authored freshness evidence for a remote
	// observation. Keep it off the wire: LastChecked remains the agent's
	// observation time, while disconnect detection must not trust agent clock
	// skew in either direction.
	ProbeReportReceivedAt time.Time `json:"-"`
}

const (
	AvailabilityAggregateHealthy     = "healthy"
	AvailabilityAggregateDegraded    = "degraded"
	AvailabilityAggregateUnavailable = "unavailable"
	AvailabilityAggregateUnknown     = "unknown"
)

// AvailabilityObservationLocationStatus is one source-owned view of a logical
// verification. LocationID is stable configuration identity; Label is resolved
// by the authenticated client so host names and network details are not copied
// into target or history persistence.
type AvailabilityObservationLocationStatus struct {
	LocationID             string    `json:"locationId"`
	Kind                   string    `json:"kind"`
	ProbeAgentID           string    `json:"probeAgentId,omitempty"`
	Outcome                string    `json:"outcome,omitempty"`
	TransportOutcome       string    `json:"transportOutcome,omitempty"`
	ApplicationOutcome     string    `json:"applicationOutcome,omitempty"`
	ApplicationStatusCode  int       `json:"applicationStatusCode,omitempty"`
	ApplicationFailureCode string    `json:"applicationFailureCode,omitempty"`
	Available              bool      `json:"available"`
	LastChecked            time.Time `json:"lastChecked,omitempty"`
	LastSuccess            time.Time `json:"lastSuccess,omitempty"`
	FreshnessAt            time.Time `json:"freshnessAt,omitempty"`
	LatencyMillis          int64     `json:"latencyMillis,omitempty"`
	ConsecutiveFailures    int       `json:"consecutiveFailures,omitempty"`
	LastError              string    `json:"lastError,omitempty"`
	Stale                  bool      `json:"stale,omitempty"`
}

// FreshnessTime returns the authoritative liveness reference for this status.
// Remote observations use server receipt time; local observations use their
// server-authored check time.
func (s AvailabilityProbeStatus) FreshnessTime() time.Time {
	if (strings.TrimSpace(s.ProbeAgentID) != "" || s.ExpectedLocations > 1) && !s.ProbeReportReceivedAt.IsZero() {
		return s.ProbeReportReceivedAt
	}
	return s.LastChecked
}

// AvailabilityProbeOutcome and its values are aliases for the shared probe
// package so existing monitoring and API callers keep their spelling.
type AvailabilityProbeOutcome = availabilityprobe.Outcome

const (
	AvailabilityProbeReachable     = availabilityprobe.OutcomeReachable
	AvailabilityProbeUnreachable   = availabilityprobe.OutcomeUnreachable
	AvailabilityProbeIndeterminate = availabilityprobe.OutcomeIndeterminate
)

type availabilityPollProvider struct{}

func newAvailabilityPollProvider() PollProvider {
	return availabilityPollProvider{}
}

func (availabilityPollProvider) Type() InstanceType {
	return InstanceTypeAvailability
}

func (availabilityPollProvider) ListInstances(m *Monitor) []string {
	targets := m.availabilityTargets()
	names := make([]string, 0, len(targets))
	for _, target := range targets {
		if !target.Enabled {
			continue
		}
		// A licensed probe assignment moves execution to the agent. When the
		// entitlement lapses the effective assignment collapses to local, and
		// the next polling cycle re-plans this instance without a restart.
		if !m.targetUsesLocalObservation(target) {
			continue
		}
		names = append(names, target.ID)
	}
	sort.Strings(names)
	return names
}

func (availabilityPollProvider) BaseInterval(m *Monitor) time.Duration {
	targets := m.availabilityTargets()
	minInterval := time.Duration(config.DefaultAvailabilityPollIntervalSecs) * time.Second
	for _, target := range targets {
		if !target.Enabled {
			continue
		}
		interval := time.Duration(target.EffectivePollIntervalSecs()) * time.Second
		if interval > 0 && interval < minInterval {
			minInterval = interval
		}
	}
	return clampInterval(minInterval, 10*time.Second, time.Hour)
}

func (availabilityPollProvider) FixedInstanceInterval(m *Monitor, instanceName string) time.Duration {
	target, ok := m.availabilityTargetByID(instanceName)
	if !ok || !target.Enabled {
		return 0
	}
	return clampInterval(time.Duration(target.EffectivePollIntervalSecs())*time.Second, 10*time.Second, time.Hour)
}

func (availabilityPollProvider) BuildPollTask(m *Monitor, instanceName string) (PollTask, error) {
	target, ok := m.availabilityTargetByID(instanceName)
	if !ok || !target.Enabled {
		return PollTask{}, fmt.Errorf("availability target %q is not enabled", instanceName)
	}
	// Refuse the local run even for a queue entry scheduled before the target
	// was assigned, so a probe-assigned target never executes twice.
	if !m.targetUsesLocalObservation(target) {
		return PollTask{}, fmt.Errorf("availability target %q has no local observation location", instanceName)
	}
	return PollTask{
		InstanceName: target.ID,
		InstanceType: string(InstanceTypeAvailability),
		Run: func(ctx context.Context) {
			m.pollAvailabilityTarget(ctx, target)
		},
	}, nil
}

func (availabilityPollProvider) DescribeInstances(m *Monitor) []PollProviderInstanceInfo {
	targets := m.availabilityTargets()
	infos := make([]PollProviderInstanceInfo, 0, len(targets))
	for _, target := range targets {
		if !target.Enabled {
			continue
		}
		infos = append(infos, PollProviderInstanceInfo{
			Name:        target.ID,
			DisplayName: target.DisplayName(),
			Connection:  availabilityConnectionKey(target.ID),
			Metadata: map[string]string{
				"address":  target.Address,
				"protocol": string(target.Protocol),
			},
		})
	}
	return infos
}

func (availabilityPollProvider) ConnectionStatuses(m *Monitor) map[string]bool {
	statuses := m.availabilityStatusSnapshotForTargets(m.availabilityTargets(), time.Now())
	out := make(map[string]bool, len(statuses))
	for targetID, status := range statuses {
		out[availabilityConnectionKey(targetID)] = status.Enabled && status.Available
	}
	return out
}

func (availabilityPollProvider) ConnectionHealthKey(_ *Monitor, instanceName string) string {
	return availabilityConnectionKey(instanceName)
}

func (availabilityPollProvider) SupplementalSource() unifiedresources.DataSource {
	return unifiedresources.SourceAvailability
}

func (availabilityPollProvider) SupplementalRecords(m *Monitor, orgID string) []unifiedresources.IngestRecord {
	targets := m.availabilityTargets()
	now := time.Now().UTC()
	statuses := m.availabilityStatusSnapshotForTargets(targets, now)
	m.syncAvailabilityProbeAlerts(targets, statuses, now)

	records := make([]unifiedresources.IngestRecord, 0, len(targets))
	for _, target := range targets {
		status := statuses[target.ID]
		if status.TargetID == "" {
			status = m.deriveAvailabilityProbeStaleness(target, availabilityStatusFromTarget(target), now)
		}
		resource, identity := availabilityResourceFromTarget(target, status, orgID, now)
		records = append(records, unifiedresources.IngestRecord{
			SourceID: target.ID,
			Resource: resource,
			Identity: identity,
		})
	}
	return records
}

func (m *Monitor) availabilityTargets() []config.AvailabilityTarget {
	if m == nil || m.configPersist == nil {
		return nil
	}
	targets, err := m.configPersist.LoadAvailabilityTargets()
	if err != nil {
		return nil
	}
	out := make([]config.AvailabilityTarget, 0, len(targets))
	for _, target := range targets {
		target = config.NormalizeAvailabilityTarget(target)
		if strings.TrimSpace(target.ID) == "" {
			continue
		}
		out = append(out, target)
	}
	sort.Slice(out, func(i, j int) bool {
		left := strings.ToLower(out[i].DisplayName())
		right := strings.ToLower(out[j].DisplayName())
		if left == right {
			return out[i].ID < out[j].ID
		}
		return left < right
	})
	return out
}

func (m *Monitor) availabilityTargetByID(id string) (config.AvailabilityTarget, bool) {
	id = strings.TrimSpace(id)
	if id == "" {
		return config.AvailabilityTarget{}, false
	}
	for _, target := range m.availabilityTargets() {
		if target.ID == id {
			return target, true
		}
	}
	return config.AvailabilityTarget{}, false
}

// AvailabilityStatusSnapshot is the single read path every availability
// consumer flows through, so probe staleness is derived here once instead of
// being re-implemented per reader.
func (m *Monitor) AvailabilityStatusSnapshot() map[string]AvailabilityProbeStatus {
	if m == nil {
		return nil
	}
	return m.availabilityStatusSnapshotForTargets(m.availabilityTargets(), time.Now())
}

func (m *Monitor) availabilityStatusSnapshotForTargets(targets []config.AvailabilityTarget, now time.Time) map[string]AvailabilityProbeStatus {
	if m == nil {
		return nil
	}
	m.mu.RLock()
	out := make(map[string]AvailabilityProbeStatus, len(m.availabilityStatuses))
	for id, status := range m.availabilityStatuses {
		out[id] = status
	}
	locationSnapshots := make(map[string]map[string]AvailabilityProbeStatus, len(m.availabilityByLocation))
	for targetID, locations := range m.availabilityByLocation {
		locationSnapshots[targetID] = make(map[string]AvailabilityProbeStatus, len(locations))
		for locationID, status := range locations {
			locationSnapshots[targetID][locationID] = status
		}
	}
	m.mu.RUnlock()

	for _, target := range targets {
		expected := m.effectiveObservationLocationIDs(target)
		locations := locationSnapshots[target.ID]
		if locations == nil {
			locations = make(map[string]AvailabilityProbeStatus)
		}
		// Adopt pre-location in-memory state into its one effective location.
		if len(locations) == 0 {
			if legacy, ok := out[target.ID]; ok && len(expected) == 1 {
				legacyLocationID := config.AvailabilityObservationLocationLocal
				if legacy.ProbeAgentID != "" {
					legacyLocationID = config.AvailabilityAgentObservationLocationID(legacy.ProbeAgentID)
				}
				if legacyLocationID == expected[0] || expected[0] == config.AvailabilityObservationLocationLocal {
					if expected[0] == config.AvailabilityObservationLocationLocal {
						legacy = adoptAvailabilityStatusAsLocal(legacy)
					}
					locations[expected[0]] = legacy
				}
			}
		}
		if len(expected) == 1 && expected[0] == config.AvailabilityObservationLocationLocal {
			if _, ok := locations[expected[0]]; !ok {
				legacy, exists := out[target.ID]
				switch {
				case !exists:
					delete(out, target.ID)
					continue
				case legacy.ExpectedLocations <= 1:
					locations[expected[0]] = adoptAvailabilityStatusAsLocal(legacy)
				default:
					// An aggregate over several locations is not one observation
					// the local location can stand in for: its freshness may be
					// one path's and its availability another's. The target
					// reads as never checked until the local poller runs, with
					// an entry of its own so connection health cannot fall back
					// to the cached state from before the lapse.
					out[target.ID] = availabilityStatusFromTarget(target)
					continue
				}
			}
		}
		for _, locationID := range expected {
			status, ok := locations[locationID]
			if !ok {
				status = availabilityStatusFromTarget(target)
				status.ProbeAgentID = config.AvailabilityObservationLocationAgentID(locationID)
			}
			if status.ProbeAgentID != "" {
				status = m.deriveAvailabilityProbeStaleness(target, status, now)
			} else if len(expected) > 1 {
				status = deriveLocalAvailabilityObservationStaleness(target, status, now)
			}
			locations[locationID] = status
		}
		out[target.ID] = aggregateAvailabilityLocationStatuses(target, expected, locations)
	}
	return out
}

// adoptAvailabilityStatusAsLocal presents a retained observation as the local
// location's once the check falls back to local observation, as it does when
// the external-probe entitlement lapses. A local location's check time is its
// freshness, but a remote observation's check time is the agent's clock, so it
// is held to the server's receipt time: a fast agent clock must not keep the
// adopted result current after its receipt is cleared.
func adoptAvailabilityStatusAsLocal(status AvailabilityProbeStatus) AvailabilityProbeStatus {
	if !status.ProbeReportReceivedAt.IsZero() && status.LastChecked.After(status.ProbeReportReceivedAt) {
		status.LastChecked = status.ProbeReportReceivedAt
	}
	status.ProbeAgentID = ""
	status.ProbeReportReceivedAt = time.Time{}
	return status
}

func (m *Monitor) RefreshAvailabilityTargets() {
	if m == nil {
		return
	}
	targets := m.availabilityTargets()
	activeIDs := make(map[string]struct{}, len(targets))
	activeProbeIDs := make(map[string]struct{}, len(targets))
	now := time.Now()
	for _, target := range targets {
		activeIDs[target.ID] = struct{}{}
		for _, locationID := range m.effectiveObservationLocationIDs(target) {
			if config.AvailabilityObservationLocationAgentID(locationID) == "" {
				continue
			}
			trackerID := target.ID + "\x00" + locationID
			activeProbeIDs[trackerID] = struct{}{}
			m.availabilityProbeAssignmentReference(target.ID, locationID, now)
		}
		if m.taskQueue == nil {
			continue
		}
		task := ScheduledTask{
			InstanceName: target.ID,
			InstanceType: InstanceTypeAvailability,
			NextRun:      now,
			Interval:     clampInterval(time.Duration(target.EffectivePollIntervalSecs())*time.Second, 10*time.Second, time.Hour),
		}
		if target.Enabled && m.targetUsesLocalObservation(target) {
			m.taskQueue.Upsert(task)
		} else {
			m.taskQueue.Remove(InstanceTypeAvailability, target.ID)
			m.removeProviderConnectionHealth(InstanceTypeAvailability, target.ID)
		}
	}

	removedIDs := make([]string, 0)
	m.mu.Lock()
	for id := range m.availabilityStatuses {
		if _, ok := activeIDs[id]; !ok {
			delete(m.availabilityStatuses, id)
			removedIDs = append(removedIDs, id)
		}
	}
	for id := range m.availabilityByLocation {
		if _, ok := activeIDs[id]; !ok {
			delete(m.availabilityByLocation, id)
		}
	}
	for id := range m.availabilityProbeTrackers {
		if _, ok := activeProbeIDs[id]; !ok {
			delete(m.availabilityProbeTrackers, id)
		}
	}
	m.mu.Unlock()
	for _, id := range removedIDs {
		m.removeProviderConnectionHealth(InstanceTypeAvailability, id)
	}

	m.refreshInstanceInfoCacheFromProviders()
	m.updateResourceStore(m.currentStateWithScope())
}

func (m *Monitor) pollAvailabilityTarget(ctx context.Context, target config.AvailabilityTarget) {
	target = config.NormalizeAvailabilityTarget(target)
	start := time.Now()
	result, err := ProbeAvailabilityTargetDetailedResult(ctx, target)
	latency := time.Since(start)
	checkedAt := time.Now().UTC()
	m.applyAvailabilityObservationDetailed(target, uuid.NewString(), checkedAt, latency, result.Outcome, result.TransportOutcome, result.Application, err, result.Certificate, "", time.Time{})
	m.updateResourceStore(m.currentStateWithScope())
}

// applyAvailabilityObservation records one availability observation regardless of
// where it executed, so remote probe results and local polls share failure
// accounting, connection health, and task bookkeeping.
func (m *Monitor) applyAvailabilityObservation(
	target config.AvailabilityTarget,
	observationID string,
	checkedAt time.Time,
	latency time.Duration,
	outcome AvailabilityProbeOutcome,
	probeErr error,
	certificate *tlsutil.CertificateObservation,
	probeAgentID string,
	probeReportReceivedAt time.Time,
) {
	m.applyAvailabilityObservationDetailed(target, observationID, checkedAt, latency, outcome, outcome, nil, probeErr, certificate, probeAgentID, probeReportReceivedAt)
}

func (m *Monitor) applyAvailabilityObservationDetailed(
	target config.AvailabilityTarget,
	observationID string,
	checkedAt time.Time,
	latency time.Duration,
	outcome AvailabilityProbeOutcome,
	transportOutcome availabilityprobe.Outcome,
	application *availabilityprobe.ApplicationResult,
	probeErr error,
	certificate *tlsutil.CertificateObservation,
	probeAgentID string,
	probeReportReceivedAt time.Time,
) {
	if transportOutcome == "" {
		transportOutcome = outcome
	}
	m.setAvailabilityStatusWithDetails(target, checkedAt, latency, outcome, transportOutcome, application, probeErr, certificate, probeAgentID, probeReportReceivedAt)
	m.recordAvailabilityHistory(target, observationID, checkedAt, latency, outcome, probeErr, probeAgentID, probeReportReceivedAt)
	current := m.AvailabilityStatusSnapshot()[target.ID]
	aggregateHealthy := current.AggregateState == AvailabilityAggregateHealthy || current.AggregateState == AvailabilityAggregateDegraded

	if aggregateHealthy {
		if m.stalenessTracker != nil {
			m.stalenessTracker.UpdateSuccess(InstanceTypeAvailability, target.ID, nil)
		}
		m.setProviderConnectionHealth(InstanceTypeAvailability, target.ID, true)
	} else {
		if m.stalenessTracker != nil {
			reason := strings.TrimSpace(current.LastError)
			if reason == "" {
				reason = "availability evidence is incomplete"
			}
			m.stalenessTracker.UpdateSuccess(InstanceTypeAvailability, target.ID, []byte(reason))
		}
		m.setProviderConnectionHealth(InstanceTypeAvailability, target.ID, false)
	}
	m.recordTaskResult(InstanceTypeAvailability, target.ID, nil)
}

func (m *Monitor) recordAvailabilityHistory(
	target config.AvailabilityTarget,
	observationID string,
	checkedAt time.Time,
	latency time.Duration,
	outcome AvailabilityProbeOutcome,
	probeErr error,
	probeAgentID string,
	probeReportReceivedAt time.Time,
) {
	if m == nil || m.metricsStore == nil {
		return
	}
	ingestedAt := time.Now().UTC()
	timelineAt := checkedAt.UTC()
	source := pkgmetrics.AvailabilitySourceLocal
	if strings.TrimSpace(probeAgentID) != "" {
		source = pkgmetrics.AvailabilitySourceAssignedAgent
		if !probeReportReceivedAt.IsZero() {
			timelineAt = probeReportReceivedAt.UTC()
			ingestedAt = timelineAt
		}
	}
	var latencyMillis *int64
	if probeErr == nil && outcome == AvailabilityProbeReachable {
		value := latency.Milliseconds()
		if value == 0 {
			value = 1
		}
		latencyMillis = &value
	}
	expected := m.effectiveObservationLocationIDs(target)
	if len(expected) > 1 {
		aggregate := m.AvailabilityStatusSnapshot()[target.ID]
		outcome = AvailabilityProbeIndeterminate
		latencyMillis = nil
		switch aggregate.AggregateState {
		case AvailabilityAggregateHealthy, AvailabilityAggregateDegraded:
			outcome = AvailabilityProbeReachable
			if aggregate.LatencyMillis > 0 {
				value := aggregate.LatencyMillis
				latencyMillis = &value
			}
		case AvailabilityAggregateUnavailable:
			outcome = AvailabilityProbeUnreachable
		}
		observationID += "|aggregate"
	}
	if err := m.metricsStore.WriteAvailabilityObservationBounded(pkgmetrics.AvailabilityObservation{
		ObservationID:   observationID,
		TargetID:        target.ID,
		ConfigRevision:  target.ConfigRevision,
		Outcome:         pkgmetrics.AvailabilityOutcome(outcome),
		ObservedAt:      checkedAt.UTC(),
		TimelineAt:      timelineAt,
		IngestedAt:      ingestedAt,
		ValidFor:        availabilityProbeStaleWindow(target),
		ExecutionSource: source,
		LatencyMillis:   latencyMillis,
	}); err != nil {
		log.Warn().Err(err).Str("target_id", target.ID).Msg("Dropping invalid availability history observation")
	}
}

func (m *Monitor) setAvailabilityStatus(
	target config.AvailabilityTarget,
	checkedAt time.Time,
	latency time.Duration,
	outcome AvailabilityProbeOutcome,
	probeErr error,
	probeAgentID string,
	probeReportReceivedAt time.Time,
) {
	m.setAvailabilityStatusWithCertificate(target, checkedAt, latency, outcome, probeErr, nil, probeAgentID, probeReportReceivedAt)
}

func (m *Monitor) setAvailabilityStatusWithCertificate(
	target config.AvailabilityTarget,
	checkedAt time.Time,
	latency time.Duration,
	outcome AvailabilityProbeOutcome,
	probeErr error,
	certificate *tlsutil.CertificateObservation,
	probeAgentID string,
	probeReportReceivedAt time.Time,
) {
	m.setAvailabilityStatusWithDetails(target, checkedAt, latency, outcome, outcome, nil, probeErr, certificate, probeAgentID, probeReportReceivedAt)
}

func (m *Monitor) setAvailabilityStatusWithDetails(
	target config.AvailabilityTarget,
	checkedAt time.Time,
	latency time.Duration,
	outcome AvailabilityProbeOutcome,
	transportOutcome availabilityprobe.Outcome,
	application *availabilityprobe.ApplicationResult,
	probeErr error,
	certificate *tlsutil.CertificateObservation,
	probeAgentID string,
	probeReportReceivedAt time.Time,
) {
	if m == nil {
		return
	}
	locationID := config.AvailabilityObservationLocationLocal
	if agentID := strings.TrimSpace(probeAgentID); agentID != "" {
		locationID = config.AvailabilityAgentObservationLocationID(agentID)
	}
	expectedLocations := m.effectiveObservationLocationIDs(target)
	status := availabilityStatusFromTarget(target)
	status.Outcome = string(outcome)
	status.TransportOutcome = string(transportOutcome)
	if application != nil {
		status.ApplicationOutcome = string(application.Outcome)
		status.ApplicationStatusCode = application.StatusCode
		status.ApplicationFailureCode = application.FailureCode
	}
	status.LastChecked = checkedAt
	status.Certificate = certificate.Clone()
	status.CertificateCurrent = status.Certificate != nil
	status.ProbeAgentID = strings.TrimSpace(probeAgentID)
	if status.ProbeAgentID != "" {
		status.ProbeReportReceivedAt = probeReportReceivedAt.UTC()
	}
	latencyMs := latency.Milliseconds()
	if probeErr == nil && latencyMs == 0 {
		latencyMs = 1
	}
	status.LatencyMillis = latencyMs
	if probeErr == nil {
		// An open-or-filtered UDP timeout is healthy probe execution but does
		// not prove endpoint reachability. Keep it non-failing without claiming
		// the endpoint is available.
		status.Available = outcome == AvailabilityProbeReachable
		if outcome == AvailabilityProbeReachable {
			status.LastSuccess = checkedAt
		}
	} else {
		status.Available = false
		status.LastError = probeErr.Error()
	}

	m.mu.Lock()
	if m.availabilityStatuses == nil {
		m.availabilityStatuses = make(map[string]AvailabilityProbeStatus)
	}
	if m.availabilityByLocation == nil {
		m.availabilityByLocation = make(map[string]map[string]AvailabilityProbeStatus)
	}
	if m.availabilityByLocation[target.ID] == nil {
		m.availabilityByLocation[target.ID] = make(map[string]AvailabilityProbeStatus)
	}
	if previous, ok := m.availabilityByLocation[target.ID][locationID]; ok {
		status.LastSuccess = previous.LastSuccess
		if status.Certificate == nil {
			status.Certificate = previous.Certificate.Clone()
		}
		if probeErr == nil {
			status.ConsecutiveFailures = 0
			status.LastError = ""
			if outcome == AvailabilityProbeReachable {
				status.LastSuccess = checkedAt
			}
		} else {
			status.ConsecutiveFailures = previous.ConsecutiveFailures + 1
		}
	} else if probeErr != nil {
		status.ConsecutiveFailures = 1
	}
	m.availabilityByLocation[target.ID][locationID] = status
	m.availabilityStatuses[target.ID] = aggregateAvailabilityLocationStatuses(
		target,
		expectedLocations,
		m.availabilityByLocation[target.ID],
	)
	m.mu.Unlock()
}

// ProbeAvailabilityTarget executes one agentless availability check. The probe
// execution core lives in internal/availabilityprobe so the host agent can run
// the same checks without importing the monitoring package; this wrapper keeps
// the historical monitoring entry point for existing callers.
func ProbeAvailabilityTarget(ctx context.Context, target config.AvailabilityTarget) error {
	return availabilityprobe.Run(ctx, target)
}

// ProbeAvailabilityTargetResult preserves UDP's open-or-filtered state rather
// than incorrectly claiming that a silent UDP endpoint was proven reachable.
func ProbeAvailabilityTargetResult(ctx context.Context, target config.AvailabilityTarget) (AvailabilityProbeOutcome, error) {
	return availabilityprobe.Result(ctx, target)
}

// ProbeAvailabilityTargetDetailedResult adds HTTPS certificate posture while
// preserving the shared reachability vocabulary.
func ProbeAvailabilityTargetDetailedResult(ctx context.Context, target config.AvailabilityTarget) (availabilityprobe.ProbeResult, error) {
	return availabilityprobe.DetailedResult(ctx, target)
}

func availabilityStatusFromTarget(target config.AvailabilityTarget) AvailabilityProbeStatus {
	return AvailabilityProbeStatus{
		TargetID:         target.ID,
		Name:             target.DisplayName(),
		TargetKind:       string(target.TargetKind),
		Address:          target.Address,
		Protocol:         string(target.Protocol),
		Enabled:          target.Enabled,
		FailureThreshold: target.EffectiveFailureThreshold(),
	}
}

func aggregateAvailabilityLocationStatuses(
	target config.AvailabilityTarget,
	expected []string,
	statuses map[string]AvailabilityProbeStatus,
) AvailabilityProbeStatus {
	aggregate := availabilityStatusFromTarget(target)
	aggregate.ExpectedLocations = len(expected)
	if len(expected) == 0 {
		aggregate.AggregateState = AvailabilityAggregateUnknown
		return aggregate
	}

	reachable, unreachable, indeterminate := 0, 0, 0
	minFailures := 0
	latestFreshness := time.Time{}
	latestChecked := time.Time{}
	latestSuccess := time.Time{}
	var representative *AvailabilityProbeStatus
	var latestReachable *AvailabilityProbeStatus
	for _, locationID := range expected {
		status, ok := statuses[locationID]
		if !ok {
			status = availabilityStatusFromTarget(target)
			status.ProbeAgentID = config.AvailabilityObservationLocationAgentID(locationID)
		}
		effectiveOutcome := status.Outcome
		if effectiveOutcome == "" && !status.LastChecked.IsZero() {
			if status.Available {
				effectiveOutcome = string(AvailabilityProbeReachable)
			} else if status.ConsecutiveFailures > 0 || strings.TrimSpace(status.LastError) != "" {
				effectiveOutcome = string(AvailabilityProbeUnreachable)
			} else {
				effectiveOutcome = string(AvailabilityProbeIndeterminate)
			}
			status.Outcome = effectiveOutcome
		}
		freshness := status.FreshnessTime()
		location := AvailabilityObservationLocationStatus{
			LocationID:             locationID,
			Kind:                   "pulse",
			ProbeAgentID:           status.ProbeAgentID,
			Outcome:                status.Outcome,
			TransportOutcome:       status.TransportOutcome,
			ApplicationOutcome:     status.ApplicationOutcome,
			ApplicationStatusCode:  status.ApplicationStatusCode,
			ApplicationFailureCode: status.ApplicationFailureCode,
			Available:              status.Available,
			LastChecked:            status.LastChecked,
			LastSuccess:            status.LastSuccess,
			FreshnessAt:            freshness,
			LatencyMillis:          status.LatencyMillis,
			ConsecutiveFailures:    status.ConsecutiveFailures,
			LastError:              status.LastError,
			Stale:                  availabilityProbeStatusIsStale(status),
		}
		if status.ProbeAgentID != "" {
			location.Kind = "agent"
		}
		aggregate.Locations = append(aggregate.Locations, location)
		if status.LastChecked.IsZero() {
			continue
		}
		if !availabilityProbeStatusIsStale(status) {
			aggregate.ReportingLocations++
		}
		if freshness.After(latestFreshness) {
			latestFreshness = freshness
		}
		if status.LastChecked.After(latestChecked) {
			latestChecked = status.LastChecked
			copy := status
			representative = &copy
		}
		if status.LastSuccess.After(latestSuccess) {
			latestSuccess = status.LastSuccess
		}
		switch effectiveOutcome {
		case string(AvailabilityProbeReachable):
			reachable++
			if latestReachable == nil || status.LastChecked.After(latestReachable.LastChecked) {
				copy := status
				latestReachable = &copy
			}
		case string(AvailabilityProbeUnreachable):
			unreachable++
			if minFailures == 0 || status.ConsecutiveFailures < minFailures {
				minFailures = status.ConsecutiveFailures
			}
		case string(AvailabilityProbeIndeterminate):
			indeterminate++
		}
	}

	if representative != nil {
		aggregate.Outcome = representative.Outcome
		aggregate.TransportOutcome = representative.TransportOutcome
		aggregate.ApplicationOutcome = representative.ApplicationOutcome
		aggregate.ApplicationStatusCode = representative.ApplicationStatusCode
		aggregate.ApplicationFailureCode = representative.ApplicationFailureCode
		aggregate.LatencyMillis = representative.LatencyMillis
		aggregate.Certificate = representative.Certificate.Clone()
		aggregate.CertificateCurrent = representative.CertificateCurrent
	}
	if latestReachable != nil {
		aggregate.LatencyMillis = latestReachable.LatencyMillis
	} else if len(expected) > 1 {
		aggregate.LatencyMillis = 0
	}
	aggregate.LastChecked = latestChecked
	aggregate.LastSuccess = latestSuccess
	aggregate.ProbeReportReceivedAt = latestFreshness
	aggregate.Disagreement = reachable > 0 && unreachable > 0
	unknown := len(expected) - reachable - unreachable - indeterminate
	switch {
	case reachable == len(expected):
		aggregate.AggregateState = AvailabilityAggregateHealthy
		aggregate.Available = true
		aggregate.Outcome = string(AvailabilityProbeReachable)
	case reachable > 0:
		aggregate.AggregateState = AvailabilityAggregateDegraded
		aggregate.Available = true
		aggregate.Outcome = string(AvailabilityProbeReachable)
	case unreachable == len(expected):
		aggregate.AggregateState = AvailabilityAggregateUnavailable
		aggregate.Available = false
		aggregate.Outcome = string(AvailabilityProbeUnreachable)
		aggregate.ConsecutiveFailures = minFailures
		aggregate.LastError = "all observation locations report the target unreachable"
	default:
		aggregate.AggregateState = AvailabilityAggregateUnknown
		aggregate.Available = false
		aggregate.Outcome = string(AvailabilityProbeIndeterminate)
		if indeterminate > 0 || unknown > 0 {
			aggregate.LastError = "observation coverage is incomplete"
		}
	}
	if len(expected) == 1 && representative != nil {
		// Preserve the established one-location response while adding the location
		// envelope, so older consumers do not lose source attribution or errors.
		locations := aggregate.Locations
		state := aggregate.AggregateState
		reporting := aggregate.ReportingLocations
		aggregate = *representative
		aggregate.AggregateState = state
		aggregate.ExpectedLocations = 1
		aggregate.ReportingLocations = reporting
		aggregate.Locations = locations
	}
	if len(expected) == 1 && representative == nil {
		status := statuses[expected[0]]
		locations := aggregate.Locations
		state := aggregate.AggregateState
		reporting := aggregate.ReportingLocations
		aggregate = status
		aggregate.AggregateState = state
		aggregate.ExpectedLocations = 1
		aggregate.ReportingLocations = reporting
		aggregate.Locations = locations
	}
	return aggregate
}

func availabilityResourceFromTarget(target config.AvailabilityTarget, status AvailabilityProbeStatus, _ string, now time.Time) (unifiedresources.Resource, unifiedresources.ResourceIdentity) {
	lastSeen := status.FreshnessTime()
	if lastSeen.IsZero() {
		lastSeen = now
	}
	resourceStatus := availabilityResourceStatus(target, status)
	data := &unifiedresources.AvailabilityData{
		TargetID:               target.ID,
		LinkedResourceID:       strings.TrimSpace(target.LinkedResourceID),
		Name:                   target.DisplayName(),
		TargetKind:             string(target.TargetKind),
		Address:                target.Address,
		Protocol:               string(target.Protocol),
		ProbeOutcome:           status.Outcome,
		TransportOutcome:       status.TransportOutcome,
		ApplicationOutcome:     status.ApplicationOutcome,
		ApplicationStatusCode:  status.ApplicationStatusCode,
		ApplicationFailureCode: status.ApplicationFailureCode,
		ProbeAgentID:           status.ProbeAgentID,
		AggregateState:         status.AggregateState,
		Disagreement:           status.Disagreement,
		ExpectedLocations:      status.ExpectedLocations,
		ReportingLocations:     status.ReportingLocations,
		UDPMode:                string(target.UDPMode),
		Port:                   target.Port,
		Path:                   target.Path,
		Enabled:                target.Enabled,
		Available:              status.Available,
		LastChecked:            timePointerIfSet(status.LastChecked),
		LastSuccess:            timePointerIfSet(status.LastSuccess),
		LatencyMillis:          status.LatencyMillis,
		ConsecutiveFailures:    status.ConsecutiveFailures,
		LastError:              status.LastError,
		FailureThreshold:       target.EffectiveFailureThreshold(),
		PollIntervalSeconds:    target.EffectivePollIntervalSecs(),
		TimeoutMillis:          target.EffectiveTimeoutMillis(),
	}
	for _, location := range status.Locations {
		data.Locations = append(data.Locations, unifiedresources.AvailabilityObservationLocation{
			LocationID:             location.LocationID,
			Kind:                   location.Kind,
			ProbeAgentID:           location.ProbeAgentID,
			Outcome:                location.Outcome,
			TransportOutcome:       location.TransportOutcome,
			ApplicationOutcome:     location.ApplicationOutcome,
			ApplicationStatusCode:  location.ApplicationStatusCode,
			ApplicationFailureCode: location.ApplicationFailureCode,
			Available:              location.Available,
			LastChecked:            timePointerIfSet(location.LastChecked),
			LastSuccess:            timePointerIfSet(location.LastSuccess),
			FreshnessAt:            timePointerIfSet(location.FreshnessAt),
			LatencyMillis:          location.LatencyMillis,
			ConsecutiveFailures:    location.ConsecutiveFailures,
			LastError:              location.LastError,
			Stale:                  location.Stale,
		})
	}
	data.CertificateMonitoring = target.CertificateMonitoringEnabled()
	data.CertificateExpiryWarningDays = target.EffectiveCertificateExpiryWarningDays()
	data.Certificate = status.Certificate.Clone()
	observedAt := status.LastChecked
	if observedAt.IsZero() {
		observedAt = lastSeen
	}
	if observedAt.After(lastSeen) {
		// A fast remote clock cannot author evidence in the server's future.
		observedAt = lastSeen
	}
	ingestedAt := now
	if !status.ProbeReportReceivedAt.IsZero() {
		ingestedAt = status.ProbeReportReceivedAt
	}
	data.Evidence = availabilityEvidenceEnvelope(target, status, observedAt, ingestedAt)
	resource := unifiedresources.Resource{
		Type:         unifiedresources.ResourceTypeNetworkEndpoint,
		Technology:   string(target.Protocol),
		Name:         target.DisplayName(),
		Status:       resourceStatus,
		LastSeen:     lastSeen,
		UpdatedAt:    now,
		Sources:      []unifiedresources.DataSource{unifiedresources.SourceAvailability},
		Tags:         availabilityResourceTags(target),
		Availability: data,
	}
	if incident := availabilityIncident(target, status, lastSeen); incident != nil {
		resource.Incidents = append(resource.Incidents, *incident)
	}
	resource.Incidents = append(resource.Incidents, availabilityCertificateIncidents(target, status)...)

	identity := unifiedresources.ResourceIdentity{}
	if ip := net.ParseIP(target.ProbeAddress()); ip != nil {
		identity.IPAddresses = []string{ip.String()}
	} else if host := target.ProbeAddress(); host != "" {
		identity.Hostnames = []string{host}
	}
	return resource, identity
}

func availabilityEvidenceEnvelope(
	target config.AvailabilityTarget,
	status AvailabilityProbeStatus,
	observedAt time.Time,
	ingestedAt time.Time,
) *operationaltrust.EvidenceEnvelope {
	if observedAt.IsZero() {
		return nil
	}
	if ingestedAt.IsZero() {
		ingestedAt = observedAt
	}

	source := operationaltrust.EvidenceSource{
		Provider:  string(unifiedresources.SourceAvailability),
		Collector: "availability-poller",
	}
	subject := operationaltrust.EvidenceSubject{
		ProviderRef:   target.ID,
		ProviderScope: "availability-target",
	}
	evidenceID, err := operationaltrust.NewEvidenceID(
		source,
		subject,
		observedAt,
		target.ID,
	)
	if err != nil {
		return nil
	}

	freshnessAt := status.FreshnessTime()
	if reachableAt := availabilityReachablePathFreshness(status); !reachableAt.IsZero() {
		// An available check's claim rests on its reachable paths alone, so
		// its evidence lasts as long as the newest of them, not as long as a
		// fresher path that failed.
		freshnessAt = reachableAt
	}
	if freshnessAt.IsZero() {
		freshnessAt = observedAt
	}
	validityWindow := time.Duration(target.EffectivePollIntervalSecs()*2) * time.Second
	if strings.TrimSpace(status.ProbeAgentID) != "" || status.ExpectedLocations > 1 {
		validityWindow = availabilityProbeStaleWindow(target)
	}
	validUntil := freshnessAt.Add(validityWindow)
	completeness := operationaltrust.EvidenceComplete
	confidence := operationaltrust.EvidenceConfirmed
	var reason *operationaltrust.EvidenceReason
	if status.LastChecked.IsZero() || status.ReportingLocations < status.ExpectedLocations {
		completeness = operationaltrust.EvidencePartial
		confidence = operationaltrust.EvidenceUnknown
		reason = &operationaltrust.EvidenceReason{
			Code:    "availability_coverage_incomplete",
			Message: "One or more observation locations have no current evidence.",
		}
	}

	envelope := operationaltrust.EvidenceEnvelope{
		ID:           evidenceID,
		Source:       source,
		Subject:      subject,
		ObservedAt:   observedAt,
		IngestedAt:   ingestedAt,
		ValidUntil:   &validUntil,
		Completeness: completeness,
		Confidence:   confidence,
		Reason:       reason,
		Permissions:  operationaltrust.EvidencePermissionsSufficient,
		PayloadRef: &operationaltrust.EvidencePayloadRef{
			Kind: "availability-target",
			ID:   target.ID,
		},
	}
	if err := envelope.Validate(); err != nil {
		return nil
	}
	return &envelope
}

// availabilityReachablePathFreshness returns the newest server-authored
// freshness among the current paths that reached an available check observed
// from several locations, and zero otherwise.
func availabilityReachablePathFreshness(status AvailabilityProbeStatus) time.Time {
	if !status.Available || len(status.Locations) < 2 {
		return time.Time{}
	}
	var latest time.Time
	for _, location := range status.Locations {
		if location.Available && !location.Stale && location.FreshnessAt.After(latest) {
			latest = location.FreshnessAt
		}
	}
	return latest
}

func availabilityResourceTags(target config.AvailabilityTarget) []string {
	tags := []string{"agentless"}
	if target.TargetKind != "" {
		tags = append(tags, string(target.TargetKind))
	}
	return tags
}

func availabilityResourceStatus(target config.AvailabilityTarget, status AvailabilityProbeStatus) unifiedresources.ResourceStatus {
	if !target.Enabled {
		return unifiedresources.StatusUnknown
	}
	if availabilityProbeStatusIsStale(status) {
		return unifiedresources.StatusWarning
	}
	switch status.AggregateState {
	case AvailabilityAggregateHealthy:
		return unifiedresources.StatusOnline
	case AvailabilityAggregateDegraded, AvailabilityAggregateUnknown:
		return unifiedresources.StatusWarning
	case AvailabilityAggregateUnavailable:
		if status.ConsecutiveFailures >= target.EffectiveFailureThreshold() {
			return unifiedresources.StatusOffline
		}
		return unifiedresources.StatusWarning
	}
	if status.LastChecked.IsZero() {
		return unifiedresources.StatusUnknown
	}
	if status.Available {
		return unifiedresources.StatusOnline
	}
	if status.ConsecutiveFailures >= target.EffectiveFailureThreshold() {
		return unifiedresources.StatusOffline
	}
	return unifiedresources.StatusWarning
}

func availabilityIncident(target config.AvailabilityTarget, status AvailabilityProbeStatus, startedAt time.Time) *unifiedresources.ResourceIncident {
	if !target.Enabled || status.Available {
		return nil
	}
	if availabilityProbeStatusIsStale(status) {
		return nil
	}
	if status.AggregateState != "" && status.AggregateState != AvailabilityAggregateUnavailable {
		return nil
	}
	if status.LastChecked.IsZero() {
		return nil
	}
	if status.ConsecutiveFailures < target.EffectiveFailureThreshold() {
		return nil
	}
	summary := fmt.Sprintf("%s is unreachable from every current observation location by %s probe", target.DisplayName(), strings.ToUpper(string(target.Protocol)))
	if status.LastError != "" {
		summary = summary + ": " + status.LastError
	}
	source := string(unifiedresources.SourceAvailability)
	if strings.TrimSpace(status.ProbeAgentID) != "" {
		source = "external-probe"
	}
	return &unifiedresources.ResourceIncident{
		Provider:  string(unifiedresources.SourceAvailability),
		NativeID:  target.ID,
		Code:      "availability_unreachable",
		Severity:  storagehealth.RiskCritical,
		Source:    source,
		Summary:   summary,
		StartedAt: startedAt,
	}
}

func availabilityCertificateIncidents(target config.AvailabilityTarget, status AvailabilityProbeStatus) []unifiedresources.ResourceIncident {
	certificate := status.Certificate
	if !target.Enabled || !target.CertificateMonitoringEnabled() || certificate == nil {
		return nil
	}
	// A retained observation remains useful in the UI during a later endpoint
	// outage, but it must not keep authoring current certificate incidents.
	if !status.CertificateCurrent || certificate.ObservedAt.IsZero() {
		return nil
	}
	observedAt := status.FreshnessTime()
	if observedAt.IsZero() {
		observedAt = certificate.ObservedAt
	}
	startedAt := observedAt
	base := target.DisplayName()
	incident := func(code string, severity storagehealth.RiskLevel, summary string) unifiedresources.ResourceIncident {
		return unifiedresources.ResourceIncident{
			Provider:  string(unifiedresources.SourceAvailability),
			NativeID:  target.ID,
			Code:      code,
			Severity:  severity,
			Source:    string(unifiedresources.SourceAvailability),
			Summary:   summary,
			StartedAt: startedAt,
		}
	}

	switch certificate.TrustStatus {
	case tlsutil.CertificateTrustExpired:
		return []unifiedresources.ResourceIncident{incident(
			"certificate_expired",
			storagehealth.RiskCritical,
			fmt.Sprintf("%s certificate expired on %s", base, certificate.NotAfter.Format("2 Jan 2006")),
		)}
	case tlsutil.CertificateTrustNotYetValid:
		return []unifiedresources.ResourceIncident{incident(
			"certificate_not_yet_valid",
			storagehealth.RiskCritical,
			fmt.Sprintf("%s certificate is not valid until %s", base, certificate.NotBefore.Format("2 Jan 2006")),
		)}
	case tlsutil.CertificateTrustUntrusted:
		summary := fmt.Sprintf("%s certificate is not trusted", base)
		if detail := strings.TrimSpace(certificate.TrustError); detail != "" {
			summary += ": " + detail
		}
		return []unifiedresources.ResourceIncident{incident("certificate_untrusted", storagehealth.RiskCritical, summary)}
	}
	if certificate.NotAfter.IsZero() {
		return nil
	}

	warningWindow := time.Duration(target.EffectiveCertificateExpiryWarningDays()) * 24 * time.Hour
	remaining := certificate.NotAfter.Sub(observedAt)
	if remaining > warningWindow {
		return nil
	}
	days := int(math.Ceil(remaining.Hours() / 24))
	if days < 0 {
		days = 0
	}
	return []unifiedresources.ResourceIncident{incident(
		"certificate_expiring",
		storagehealth.RiskWarning,
		fmt.Sprintf("%s certificate expires in %d days on %s", base, days, certificate.NotAfter.Format("2 Jan 2006")),
	)}
}

func availabilityConnectionKey(targetID string) string {
	targetID = strings.TrimSpace(targetID)
	if targetID == "" {
		return ""
	}
	return "availability-" + targetID
}