Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

27.2 KB
Raw Blame
package ai

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"math"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
	"github.com/rcourtman/pulse-go-rewrite/pkg/aicontracts"
	"github.com/rs/zerolog/log"
)

const (
	patrolObserverResourceStateFormat    = "pulse-resource-state/v1"
	patrolObserverAvailabilityFormat     = "pulse-availability-state/v1"
	patrolObserverResourceMetricFormat   = "pulse-resource-metric/v1"
	patrolObserverSweepInterval          = 5 * time.Second
	patrolObserverMinSampleInterval      = 10 * time.Second
	patrolObserverMaxSampleInterval      = 5 * time.Minute
	patrolObserverMinLease               = 2 * time.Minute
	patrolObserverMaxLease               = 30 * time.Minute
	patrolObserverMaxConsecutiveFailures = 10
	patrolObserverWakeRetryInterval      = 15 * time.Minute
)

type patrolResourceStateProbe struct {
	Runtime                      string `json:"runtime"`
	Path                         string `json:"path"`
	Operator                     string `json:"operator"`
	Value                        string `json:"value"`
	SampleIntervalSeconds        int    `json:"sample_interval_seconds"`
	WakeAfterConsecutiveFailures int    `json:"wake_after_consecutive_failures"`
}

type patrolAvailabilityStateProbe struct {
	Runtime                      string `json:"runtime"`
	TargetID                     string `json:"target_id"`
	Path                         string `json:"path"`
	Operator                     string `json:"operator"`
	Value                        string `json:"value"`
	SampleIntervalSeconds        int    `json:"sample_interval_seconds"`
	WakeAfterConsecutiveFailures int    `json:"wake_after_consecutive_failures"`
}

type patrolResourceMetricProbe struct {
	Runtime                      string  `json:"runtime"`
	Metric                       string  `json:"metric"`
	Operator                     string  `json:"operator"`
	Threshold                    float64 `json:"threshold"`
	SampleIntervalSeconds        int     `json:"sample_interval_seconds"`
	WakeAfterConsecutiveFailures int     `json:"wake_after_consecutive_failures"`
	MaxEvidenceAgeSeconds        int     `json:"max_evidence_age_seconds"`
}

type patrolValidatedObserverProbe struct {
	runtime                      string
	targetID                     string
	path                         string
	operator                     string
	value                        string
	metric                       string
	threshold                    float64
	maxEvidenceAgeSeconds        int
	sampleIntervalSeconds        int
	wakeAfterConsecutiveFailures int
}

type patrolObserverValidationError struct{ code string }

func (e *patrolObserverValidationError) Error() string { return e.code }

type patrolObserverExecution struct {
	nextDue             time.Time
	consecutiveFailures int
	wakeEmitted         bool
	lastWakeAt          time.Time
	deliveryConfirmed   bool
}

type patrolObserverRuntime struct {
	mu         sync.Mutex
	executions map[string]patrolObserverExecution
}

func newPatrolObserverRuntime() *patrolObserverRuntime {
	return &patrolObserverRuntime{
		executions: make(map[string]patrolObserverExecution),
	}
}

func (p *PatrolService) objectiveObserverLoop(ctx context.Context) {
	ticker := time.NewTicker(patrolObserverSweepInterval)
	defer ticker.Stop()
	p.processObjectiveObservers(time.Now().UTC())
	for {
		select {
		case <-ctx.Done():
			return
		case <-p.stopCh:
			return
		case now := <-ticker.C:
			p.processObjectiveObservers(now.UTC())
		}
	}
}

func (p *PatrolService) processObjectiveObservers(now time.Time) {
	if p == nil {
		return
	}
	p.mu.RLock()
	store := p.objectiveStore
	runtime := p.observerRuntime
	p.mu.RUnlock()
	if store == nil || runtime == nil {
		return
	}
	activeExecutions := make(map[string]struct{})
	healthUpdates := make([]patrolObserverHealthUpdate, 0)
	objectives := store.List(false, now)
	p.queueMissingObjectiveCoverage(objectives, now)
	for _, objective := range objectives {
		if objective.Status != PatrolObjectiveActive || objective.Observer == nil {
			continue
		}
		activeExecutions[fmt.Sprintf("%s/%d", objective.Observer.ID, objective.Observer.Version)] = struct{}{}
		if update := p.reconcileObjectiveObserver(store, runtime, objective, now); update != nil {
			healthUpdates = append(healthUpdates, *update)
		}
	}
	runtime.retain(activeExecutions)
	if _, err := store.RefreshObserverHealthBatch(healthUpdates, now); err != nil {
		log.Warn().Err(err).Int("lease_count", len(healthUpdates)).Msg("Patrol observer health lease batch failed")
	}
}

func (r *patrolObserverRuntime) retain(active map[string]struct{}) {
	r.mu.Lock()
	defer r.mu.Unlock()
	for key := range r.executions {
		if _, ok := active[key]; !ok {
			delete(r.executions, key)
		}
	}
}

func (p *PatrolService) reconcileObjectiveObserver(store *PatrolObjectiveStore, runtime *patrolObserverRuntime, objective PatrolObjective, now time.Time) *patrolObserverHealthUpdate {
	artifact, ok := store.GetObserverArtifact(objective.ID)
	if !ok {
		p.recordObserverValidationFailure(store, objective, "observer_artifact_missing", now)
		return nil
	}
	state := p.currentPatrolRuntimeState()
	effectiveObjective := objective
	if len(effectiveObjective.Scope.ResourceIDs) == 0 {
		effectiveObjective.Scope.ResourceIDs = patrolObjectiveEffectiveResourceIDs(state)
	}
	probe, validationErr := validatePatrolObserverArtifact(effectiveObjective, artifact)
	if validationErr != nil {
		p.recordObserverValidationFailure(store, objective, validationErr.code, now)
		return nil
	}
	if bindingErr := p.validatePatrolObserverBinding(effectiveObjective, probe, state); bindingErr != nil {
		p.recordObserverValidationFailure(store, objective, bindingErr.code, now)
		return nil
	}

	for objective.Observer != nil && (objective.Observer.State == PatrolObserverProposed || objective.Observer.State == PatrolObserverRejected || objective.Observer.State == PatrolObserverValidated || objective.Observer.State == PatrolObserverDegraded) {
		next := *clonePatrolObserver(objective.Observer)
		next.Artifact = &artifact
		switch objective.Observer.State {
		case PatrolObserverProposed:
			next.State = PatrolObserverValidated
			next.FailureCode = ""
		case PatrolObserverRejected:
			next.State = PatrolObserverValidated
			next.FailureCode = ""
		case PatrolObserverValidated:
			next.State = PatrolObserverInstalled
			next.ValidUntil = nil
			next.LastEvidenceAt = nil
		case PatrolObserverDegraded:
			next.State = PatrolObserverInstalled
			next.FailureCode = ""
			next.ValidUntil = nil
			next.LastEvidenceAt = nil
		}
		updated, err := store.RecordObserver(objective.ID, objective.Revision, next, "patrol:observer-runtime", now)
		if err != nil {
			if !errors.Is(err, ErrPatrolObjectiveConflict) {
				log.Warn().Err(err).Str("objective_id", objective.ID).Msg("Patrol observer lifecycle transition failed")
			}
			return nil
		}
		objective = updated
	}
	if objective.Observer == nil || (objective.Observer.State != PatrolObserverInstalled && objective.Observer.State != PatrolObserverDegraded) {
		return nil
	}
	effectiveObjective = objective
	if len(effectiveObjective.Scope.ResourceIDs) == 0 {
		effectiveObjective.Scope.ResourceIDs = patrolObjectiveEffectiveResourceIDs(state)
	}
	return p.evaluateObjectiveObserver(runtime, effectiveObjective, probe, state, now)
}

func (p *PatrolService) recordObserverValidationFailure(store *PatrolObjectiveStore, objective PatrolObjective, code string, now time.Time) {
	if objective.Observer == nil || objective.Observer.State == PatrolObserverDisabled || objective.Observer.FailureCode == code {
		return
	}
	artifact, _ := store.GetObserverArtifact(objective.ID)
	next := *clonePatrolObserver(objective.Observer)
	next.FailureCode = code
	if next.State == PatrolObserverProposed || next.State == PatrolObserverRejected {
		next.State = PatrolObserverRejected
	} else {
		next.State = PatrolObserverDegraded
		next.ValidUntil = nil
	}
	if artifact.Format != "" {
		next.Artifact = &artifact
	}
	if _, err := store.RecordObserver(objective.ID, objective.Revision, next, "patrol:observer-validator", now); err != nil && !errors.Is(err, ErrPatrolObjectiveConflict) {
		log.Warn().Err(err).Str("objective_id", objective.ID).Str("failure_code", code).Msg("Patrol observer validation failure could not be recorded")
	}
}

func validatePatrolObserverArtifact(objective PatrolObjective, artifact PatrolObserverArtifact) (patrolValidatedObserverProbe, *patrolObserverValidationError) {
	fail := func(code string) (patrolValidatedObserverProbe, *patrolObserverValidationError) {
		return patrolValidatedObserverProbe{}, &patrolObserverValidationError{code: code}
	}
	if objective.Observer == nil || !objective.Observer.ReadOnly {
		return fail("observer_not_read_only")
	}
	if len(objective.Scope.ResourceIDs) == 0 {
		return fail("observer_scope_required")
	}
	if len(objective.Observer.TriggerKinds) != 1 || objective.Observer.TriggerKinds[0] != PatrolObserverTriggerInterval {
		return fail("observer_trigger_unsupported")
	}
	var requirements map[string]json.RawMessage
	if err := decodeStrictJSONObject(artifact.Requirements, &requirements); err != nil {
		return fail("observer_requirements_invalid")
	}
	if len(requirements) != 0 {
		return fail("observer_requirements_unsupported")
	}
	var envelope struct {
		Runtime string `json:"runtime"`
	}
	if err := json.Unmarshal(artifact.Probe, &envelope); err != nil {
		return fail("observer_probe_invalid")
	}
	var probe patrolValidatedObserverProbe
	switch strings.TrimSpace(envelope.Runtime) {
	case patrolObserverResourceStateFormat:
		var decoded patrolResourceStateProbe
		if err := decodeStrictJSONObject(artifact.Probe, &decoded); err != nil {
			return fail("observer_probe_invalid")
		}
		probe = patrolValidatedObserverProbe{
			runtime: decoded.Runtime, path: decoded.Path, operator: decoded.Operator,
			value: decoded.Value, sampleIntervalSeconds: decoded.SampleIntervalSeconds,
			wakeAfterConsecutiveFailures: decoded.WakeAfterConsecutiveFailures,
		}
		if probe.path != "status" {
			return fail("observer_path_unsupported")
		}
		switch unifiedresources.ResourceStatus(strings.ToLower(strings.TrimSpace(probe.value))) {
		case unifiedresources.StatusOnline, unifiedresources.StatusOffline, unifiedresources.StatusWarning, unifiedresources.StatusUnknown:
			probe.value = strings.ToLower(strings.TrimSpace(probe.value))
		default:
			return fail("observer_value_unsupported")
		}
	case patrolObserverAvailabilityFormat:
		var decoded patrolAvailabilityStateProbe
		if err := decodeStrictJSONObject(artifact.Probe, &decoded); err != nil {
			return fail("observer_probe_invalid")
		}
		probe = patrolValidatedObserverProbe{
			runtime: decoded.Runtime, targetID: strings.TrimSpace(decoded.TargetID), path: decoded.Path,
			operator: decoded.Operator, value: strings.ToLower(strings.TrimSpace(decoded.Value)),
			sampleIntervalSeconds:        decoded.SampleIntervalSeconds,
			wakeAfterConsecutiveFailures: decoded.WakeAfterConsecutiveFailures,
		}
		if probe.targetID == "" {
			return fail("observer_availability_target_required")
		}
		if probe.path != "probe_outcome" {
			return fail("observer_path_unsupported")
		}
		switch probe.value {
		case "reachable", "unreachable", "indeterminate":
		default:
			return fail("observer_value_unsupported")
		}
	case patrolObserverResourceMetricFormat:
		var decoded patrolResourceMetricProbe
		if err := decodeStrictJSONObject(artifact.Probe, &decoded); err != nil {
			return fail("observer_probe_invalid")
		}
		probe = patrolValidatedObserverProbe{
			runtime: decoded.Runtime, metric: strings.ToLower(strings.TrimSpace(decoded.Metric)),
			operator: strings.ToLower(strings.TrimSpace(decoded.Operator)), threshold: decoded.Threshold,
			sampleIntervalSeconds:        decoded.SampleIntervalSeconds,
			wakeAfterConsecutiveFailures: decoded.WakeAfterConsecutiveFailures,
			maxEvidenceAgeSeconds:        decoded.MaxEvidenceAgeSeconds,
		}
		switch probe.metric {
		case "cpu_percent", "memory_percent", "disk_percent", "temperature_celsius":
		default:
			return fail("observer_metric_unsupported")
		}
		if math.IsNaN(probe.threshold) || math.IsInf(probe.threshold, 0) {
			return fail("observer_threshold_invalid")
		}
		if probe.metric == "temperature_celsius" {
			if probe.threshold < -100 || probe.threshold > 300 {
				return fail("observer_threshold_out_of_bounds")
			}
		} else if probe.threshold < 0 || probe.threshold > 100 {
			return fail("observer_threshold_out_of_bounds")
		}
		if probe.maxEvidenceAgeSeconds < probe.sampleIntervalSeconds || probe.maxEvidenceAgeSeconds > 3600 {
			return fail("observer_evidence_age_out_of_bounds")
		}
	default:
		return fail("observer_runtime_unsupported")
	}
	if probe.runtime == patrolObserverResourceMetricFormat {
		switch probe.operator {
		case "less_than", "less_than_or_equals", "greater_than", "greater_than_or_equals":
		default:
			return fail("observer_operator_unsupported")
		}
	} else if probe.operator != "equals" && probe.operator != "not_equals" {
		return fail("observer_operator_unsupported")
	}
	interval := time.Duration(probe.sampleIntervalSeconds) * time.Second
	if interval < patrolObserverMinSampleInterval || interval > patrolObserverMaxSampleInterval {
		return fail("observer_interval_out_of_bounds")
	}
	if probe.wakeAfterConsecutiveFailures < 1 || probe.wakeAfterConsecutiveFailures > patrolObserverMaxConsecutiveFailures {
		return fail("observer_failure_window_out_of_bounds")
	}
	return probe, nil
}

func (p *PatrolService) validatePatrolObserverBinding(objective PatrolObjective, probe patrolValidatedObserverProbe, state patrolRuntimeState) *patrolObserverValidationError {
	switch probe.runtime {
	case patrolObserverAvailabilityFormat:
		check, ownerID, found := patrolAvailabilityCheckByTarget(state, probe.targetID)
		if !found {
			return &patrolObserverValidationError{code: "observer_availability_target_missing"}
		}
		if !check.Enabled {
			return &patrolObserverValidationError{code: "observer_availability_target_disabled"}
		}
		scope := make(map[string]struct{}, len(objective.Scope.ResourceIDs))
		for _, resourceID := range objective.Scope.ResourceIDs {
			scope[canonicalPatrolScopeToken(resourceID)] = struct{}{}
		}
		_, ownerInScope := scope[canonicalPatrolScopeToken(ownerID)]
		_, linkedInScope := scope[canonicalPatrolScopeToken(check.LinkedResourceID)]
		if !ownerInScope && !linkedInScope {
			return &patrolObserverValidationError{code: "observer_availability_target_out_of_scope"}
		}
		return nil
	default:
		return nil
	}
}

func decodeStrictJSONObject(data []byte, target interface{}) error {
	decoder := json.NewDecoder(bytes.NewReader(data))
	decoder.DisallowUnknownFields()
	if err := decoder.Decode(target); err != nil {
		return err
	}
	if err := decoder.Decode(&struct{}{}); err != io.EOF {
		return fmt.Errorf("expected one JSON object")
	}
	return nil
}

func (p *PatrolService) evaluateObjectiveObserver(runtime *patrolObserverRuntime, objective PatrolObjective, probe patrolValidatedObserverProbe, state patrolRuntimeState, now time.Time) *patrolObserverHealthUpdate {
	observer := objective.Observer
	key := fmt.Sprintf("%s/%d", observer.ID, observer.Version)
	runtime.mu.Lock()
	execution := runtime.executions[key]
	if !execution.nextDue.IsZero() && now.Before(execution.nextDue) {
		runtime.mu.Unlock()
		return nil
	}
	interval := time.Duration(probe.sampleIntervalSeconds) * time.Second
	execution.nextDue = now.Add(interval)
	runtime.mu.Unlock()
	failing := make([]string, 0)
	evidenceDetails := make([]string, 0)
	if probe.runtime == patrolObserverAvailabilityFormat {
		check, ownerID, exists := patrolAvailabilityCheckByTarget(state, probe.targetID)
		matched := exists && check.LastChecked != nil && strings.EqualFold(check.ProbeOutcome, probe.value)
		if probe.operator == "not_equals" {
			matched = exists && check.LastChecked != nil && !strings.EqualFold(check.ProbeOutcome, probe.value)
		}
		if !matched {
			failing = append(failing, ownerID)
		}
	} else if probe.runtime == patrolObserverResourceMetricFormat {
		resources := patrolUnifiedResourcesByID(state)
		maxAge := time.Duration(probe.maxEvidenceAgeSeconds) * time.Second
		for _, resourceID := range objective.Scope.ResourceIDs {
			resource, exists := resources[canonicalPatrolScopeToken(resourceID)]
			if !exists {
				failing = append(failing, resourceID)
				evidenceDetails = append(evidenceDetails, resourceID+"=resource_missing")
				continue
			}
			observedAt := resource.LastSeen
			if observedAt.IsZero() {
				observedAt = resource.UpdatedAt
			}
			if probe.metric == "memory_percent" && (resource.Type == unifiedresources.ResourceTypeVM || resource.Type == unifiedresources.ResourceTypeSystemContainer) {
				// The row can be refreshed by a different source. Only the selected
				// guest memory sample owns its freshness and pressure semantics.
				memory := unifiedresources.GuestMemoryEvidenceForResource(&resource, now)
				if !memory.PressureKnown {
					failing = append(failing, resourceID)
					evidenceDetails = append(evidenceDetails, resourceID+"=guest_memory_pressure_unknown")
					continue
				}
				if memory.State != "" {
					observedAt = memory.ObservedAt
				}
			}
			if observedAt.IsZero() || now.Sub(observedAt) > maxAge {
				failing = append(failing, resourceID)
				evidenceDetails = append(evidenceDetails, resourceID+"=metric_stale")
				continue
			}
			value, available := patrolResourceMetricValue(resource, probe.metric)
			if !available {
				failing = append(failing, resourceID)
				evidenceDetails = append(evidenceDetails, resourceID+"=metric_missing")
				continue
			}
			if !patrolMetricPredicate(value, probe.operator, probe.threshold) {
				failing = append(failing, resourceID)
				evidenceDetails = append(evidenceDetails, fmt.Sprintf("%s=%.2f", resourceID, value))
			}
		}
	} else {
		statuses := patrolRuntimeCanonicalStatuses(state)
		for _, resourceID := range objective.Scope.ResourceIDs {
			status, exists := statuses[canonicalPatrolScopeToken(resourceID)]
			matched := exists && string(status) == probe.value
			if probe.operator == "not_equals" {
				matched = exists && string(status) != probe.value
			}
			if !matched {
				failing = append(failing, resourceID)
			}
		}
	}

	return p.finalizeObjectiveObserverSample(runtime, objective, probe, failing, evidenceDetails, now)
}

func (p *PatrolService) finalizeObjectiveObserverSample(runtime *patrolObserverRuntime, objective PatrolObjective, probe patrolValidatedObserverProbe, failing, evidenceDetails []string, now time.Time) *patrolObserverHealthUpdate {
	observer := objective.Observer
	key := fmt.Sprintf("%s/%d", observer.ID, observer.Version)
	interval := time.Duration(probe.sampleIntervalSeconds) * time.Second
	runtime.mu.Lock()
	execution := runtime.executions[key]
	if len(failing) == 0 {
		execution.consecutiveFailures = 0
		execution.wakeEmitted = false
		execution.lastWakeAt = time.Time{}
		execution.deliveryConfirmed = false
	} else {
		execution.consecutiveFailures++
	}
	runtime.executions[key] = execution
	runtime.mu.Unlock()

	if len(failing) > 0 && execution.wakeEmitted && !execution.deliveryConfirmed &&
		p.objectiveWakeDelivered(objective, execution.lastWakeAt) {
		runtime.mu.Lock()
		latest := runtime.executions[key]
		if latest.lastWakeAt.Equal(execution.lastWakeAt) {
			latest.deliveryConfirmed = true
			runtime.executions[key] = latest
			execution = latest
		}
		runtime.mu.Unlock()
	}
	shouldWake := len(failing) > 0 && execution.consecutiveFailures >= probe.wakeAfterConsecutiveFailures &&
		(!execution.wakeEmitted || (!execution.deliveryConfirmed && now.Sub(execution.lastWakeAt) >= patrolObserverWakeRetryInterval))

	leaseDuration := 3 * interval
	if leaseDuration < patrolObserverMinLease {
		leaseDuration = patrolObserverMinLease
	}
	if leaseDuration > patrolObserverMaxLease {
		leaseDuration = patrolObserverMaxLease
	}
	refreshLease := observer.ValidUntil == nil || observer.ValidUntil.Sub(now) <= leaseDuration/2
	var healthUpdate *patrolObserverHealthUpdate
	if refreshLease {
		healthUpdate = &patrolObserverHealthUpdate{
			ObjectiveID: objective.ID,
			ObserverID:  observer.ID,
			Version:     observer.Version,
			ValidUntil:  now.Add(leaseDuration),
			EvidenceAt:  now,
		}
	}
	if !shouldWake {
		return healthUpdate
	}
	sort.Strings(failing)
	evidence := patrolObserverEvidence(probe, failing, evidenceDetails, len(objective.Scope.ResourceIDs), execution.consecutiveFailures)
	scope := PatrolScope{
		ResourceIDs: failing,
		Depth:       PatrolDepthQuick,
		Reason:      TriggerReasonObjectiveEvidence,
		Priority:    triggerPriorityObjective,
		Context:     fmt.Sprintf("Local observer %s for objective %s detected %d of %d scoped resources outside its canonical status predicate.", observer.ID, objective.ID, len(failing), len(objective.Scope.ResourceIDs)),
		ObjectiveContext: &aicontracts.PatrolObjectiveContext{
			ObjectiveID:         objective.ID,
			Revision:            objective.Revision,
			Brief:               objective.Brief,
			Context:             objective.OptionalContext,
			ObserverID:          observer.ID,
			ObserverVersion:     observer.Version,
			ObservedResourceIDs: append([]string(nil), failing...),
			Evidence:            evidence,
			ObservedAt:          now,
		},
	}
	p.mu.RLock()
	tm := p.triggerManager
	p.mu.RUnlock()
	accepted := tm != nil && tm.TriggerPatrol(scope)
	if accepted {
		runtime.mu.Lock()
		execution = runtime.executions[key]
		execution.wakeEmitted = true
		execution.lastWakeAt = now
		execution.deliveryConfirmed = false
		runtime.executions[key] = execution
		runtime.mu.Unlock()
		log.Info().Str("objective_id", objective.ID).Int("affected_resources", len(failing)).Msg("Patrol objective observer queued an evidence-triggered check")
	}
	return healthUpdate
}

func patrolObserverEvidence(probe patrolValidatedObserverProbe, failing, details []string, scopedResources, consecutiveFailures int) string {
	if probe.runtime == patrolObserverAvailabilityFormat {
		return fmt.Sprintf(
			"Canonical availability target %s did not satisfy %s %s after %d consecutive local samples.",
			probe.targetID, probe.operator, probe.value, consecutiveFailures,
		)
	}
	if probe.runtime == patrolObserverResourceMetricFormat {
		detail := ""
		if len(details) > 0 {
			detail = " Evidence: " + strings.Join(details, ", ") + "."
		}
		return fmt.Sprintf(
			"Canonical %s did not satisfy %s %.2f for %d of %d scoped resources after %d consecutive local samples.%s",
			probe.metric, probe.operator, probe.threshold, len(failing), scopedResources, consecutiveFailures, detail,
		)
	}
	return fmt.Sprintf(
		"Canonical resource status did not satisfy %s %s for %d of %d scoped resources after %d consecutive local samples.",
		probe.operator, probe.value, len(failing), scopedResources, consecutiveFailures,
	)
}

func patrolUnifiedResourcesByID(state patrolRuntimeState) map[string]unifiedresources.Resource {
	result := make(map[string]unifiedresources.Resource)
	if state.unifiedResourceProvider == nil {
		return result
	}
	for _, resource := range state.unifiedResourceProvider.GetAll() {
		if token := canonicalPatrolScopeToken(resource.ID); token != "" {
			result[token] = resource
		}
	}
	return result
}

func patrolObjectiveEffectiveResourceIDs(state patrolRuntimeState) []string {
	resources := patrolUnifiedResourcesByID(state)
	result := make([]string, 0, len(resources))
	for _, resource := range resources {
		if id := strings.TrimSpace(resource.ID); id != "" {
			result = append(result, id)
		}
	}
	sort.Strings(result)
	return result
}

func patrolResourceMetricValue(resource unifiedresources.Resource, metric string) (float64, bool) {
	if metric == "temperature_celsius" {
		if resource.Temperature == nil || math.IsNaN(*resource.Temperature) || math.IsInf(*resource.Temperature, 0) {
			return 0, false
		}
		return *resource.Temperature, true
	}
	if resource.Metrics == nil {
		return 0, false
	}
	var value *unifiedresources.MetricValue
	switch metric {
	case "cpu_percent":
		value = resource.Metrics.CPU
	case "memory_percent":
		value = resource.Metrics.Memory
	case "disk_percent":
		value = resource.Metrics.Disk
	}
	if value == nil || math.IsNaN(value.Percent) || math.IsInf(value.Percent, 0) {
		return 0, false
	}
	return value.Percent, true
}

func patrolMetricPredicate(value float64, operator string, threshold float64) bool {
	switch operator {
	case "less_than":
		return value < threshold
	case "less_than_or_equals":
		return value <= threshold
	case "greater_than":
		return value > threshold
	case "greater_than_or_equals":
		return value >= threshold
	default:
		return false
	}
}

func patrolAvailabilityCheckByTarget(state patrolRuntimeState, targetID string) (unifiedresources.AvailabilityData, string, bool) {
	targetID = strings.TrimSpace(targetID)
	if targetID == "" || state.unifiedResourceProvider == nil {
		return unifiedresources.AvailabilityData{}, "", false
	}
	for _, resource := range state.unifiedResourceProvider.GetAll() {
		for _, check := range unifiedresources.AvailabilityChecksForResource(resource) {
			if strings.TrimSpace(check.TargetID) == targetID {
				return check, resource.ID, true
			}
		}
	}
	return unifiedresources.AvailabilityData{}, "", false
}

func (p *PatrolService) objectiveWakeDelivered(objective PatrolObjective, lastWakeAt time.Time) bool {
	if p == nil || lastWakeAt.IsZero() || objective.Observer == nil || p.runHistoryStore == nil {
		return false
	}
	for _, run := range p.runHistoryStore.GetAll() {
		if run.CompletedAt.Before(lastWakeAt) || run.ErrorCount > 0 || strings.EqualFold(strings.TrimSpace(run.Status), "error") {
			continue
		}
		context := run.ObjectiveContext
		if context == nil {
			continue
		}
		if context.ObjectiveID == objective.ID && context.Revision == objective.Revision &&
			context.ObserverID == objective.Observer.ID && context.ObserverVersion == objective.Observer.Version {
			return true
		}
	}
	return false
}

func patrolRuntimeCanonicalStatuses(state patrolRuntimeState) map[string]unifiedresources.ResourceStatus {
	result := make(map[string]unifiedresources.ResourceStatus)
	add := func(status unifiedresources.ResourceStatus, ids ...string) {
		for _, id := range ids {
			if token := canonicalPatrolScopeToken(id); token != "" {
				result[token] = normalizePatrolResourceStatus(string(status))
			}
		}
	}
	if state.readState == nil {
		state = state.withDerivedProviders()
	}
	rs := state.readState
	if rs == nil {
		return result
	}
	for _, v := range rs.VMs() {
		add(v.Status(), v.ID(), v.SourceID())
	}
	for _, v := range rs.Containers() {
		add(v.Status(), v.ID(), v.SourceID())
	}
	for _, v := range rs.Nodes() {
		add(v.Status(), v.ID(), v.SourceID())
	}
	for _, v := range rs.Hosts() {
		add(v.Status(), v.ID())
	}
	for _, v := range rs.DockerHosts() {
		add(v.Status(), v.ID(), v.HostSourceID())
	}
	for _, v := range rs.DockerContainers() {
		add(v.Status(), v.ID(), v.ContainerID())
	}
	for _, v := range rs.StoragePools() {
		add(v.Status(), v.ID(), v.SourceID())
	}
	for _, v := range rs.PhysicalDisks() {
		add(v.Status(), v.ID())
	}
	for _, v := range rs.PBSInstances() {
		add(v.Status(), v.ID())
	}
	for _, v := range rs.PMGInstances() {
		add(v.Status(), v.ID())
	}
	for _, v := range rs.K8sClusters() {
		add(v.Status(), v.ID())
	}
	return result
}

func normalizePatrolResourceStatus(value string) unifiedresources.ResourceStatus {
	switch strings.ToLower(strings.TrimSpace(value)) {
	case "online", "running", "active", "healthy", "up", "available", "ready":
		return unifiedresources.StatusOnline
	case "offline", "stopped", "inactive", "failed", "error", "down", "unavailable":
		return unifiedresources.StatusOffline
	case "warning", "degraded", "unhealthy":
		return unifiedresources.StatusWarning
	default:
		return unifiedresources.StatusUnknown
	}
}