Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

108 KB
Raw Blame
// patrol_run.go implements the PatrolService runtime: Start/Stop lifecycle,
// the main patrol loop, scoped patrol execution, alert auto-resolution,
// live streaming to UI subscribers, and run history tracking.
package ai

import (
	"context"
	"crypto/sha256"
	"errors"
	"fmt"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/google/uuid"
	"github.com/rcourtman/pulse-go-rewrite/internal/ai/circuit"
	"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
	"github.com/rs/zerolog/log"
)

// Patrol run lifecycle constants.
const (
	initialPatrolStartDelay   = 30 * time.Second // Delay before first patrol after startup
	findingCleanupAge         = 24 * time.Hour   // Resolved findings older than this are purged
	scopedPatrolRetryBackoff1 = 5 * time.Second  // First retry backoff for dropped scoped patrols
	scopedPatrolRetryBackoff2 = 15 * time.Second // Second retry backoff for dropped scoped patrols
	scopedPatrolMaxRetries    = 2                // Maximum re-queue attempts for dropped scoped patrols
	scopedPatrolLogIDLimit    = 10               // Maximum number of effective scope IDs to log inline
	remindSweepInterval       = time.Hour        // Cadence for the proactive will_fix_later remind sweep
	remindSweepInitialDelay   = 30 * time.Second // First sweep runs shortly after start so a restart-with-overdue does not have to wait a full hour
)

// Start begins the background patrol loop
func (p *PatrolService) Start(ctx context.Context) {
	p.mu.Lock()
	if p.running {
		p.mu.Unlock()
		return
	}
	p.running = true
	p.stopCh = make(chan struct{})
	p.configChanged = make(chan struct{}, 1) // Buffered to allow non-blocking send
	p.mu.Unlock()

	log.Info().
		Dur("interval", p.config.GetInterval()).
		Msg("Starting AI Patrol Service")

	go p.patrolLoop(ctx)
	go p.remindSweepLoop(ctx)
	go p.objectiveObserverLoop(ctx)
}

// Stop stops the patrol service. It signals the patrol loop to exit, then
// waits up to 15 seconds for in-flight investigations to finish and
// force-saves findings/investigation state to disk.
func (p *PatrolService) Stop() {
	p.mu.Lock()
	if !p.running {
		p.mu.Unlock()
		return
	}
	p.running = false
	close(p.stopCh)
	orchestrator := p.investigationOrchestrator
	findings := p.findings
	p.mu.Unlock()

	log.Info().Msg("stopping AI Patrol Service")

	// Give investigations 15 seconds to finish (leaves headroom within server's 30s budget)
	shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 15*time.Second)
	defer shutdownCancel()

	// Signal orchestrator to cancel running investigations and persist state
	if orchestrator != nil {
		if err := orchestrator.Shutdown(shutdownCtx); err != nil {
			log.Warn().Err(err).Msg("AI Patrol: Investigation orchestrator shutdown returned error")
		}
	}

	// Wait for investigation goroutines tracked by PatrolService
	done := make(chan struct{})
	go func() {
		p.investigationWg.Wait()
		close(done)
	}()
	select {
	case <-done:
		// All investigation goroutines finished
	case <-shutdownCtx.Done():
		log.Warn().Msg("AI Patrol: Timed out waiting for investigation goroutines to finish")
	}

	// Force-save findings store
	if findings != nil {
		if err := findings.ForceSave(); err != nil {
			log.Error().Err(err).Msg("AI Patrol: Failed to force-save findings during shutdown")
		}
	}

	log.Info().Msg("AI Patrol Service stopped")
}

// patrolLoop is the main background loop
func (p *PatrolService) patrolLoop(ctx context.Context) {
	// Seed recency from persisted run history so the API can return Patrol timing
	// metadata immediately (before the first in-process patrol completes).
	if history := p.GetRunHistory(10); len(history) > 0 {
		lastActivity, lastFullPatrol := patrolRecencyFromHistory(history)
		p.mu.Lock()
		p.lastActivity = lastActivity
		p.lastFullPatrol = lastFullPatrol
		p.mu.Unlock()
	}

	// Let configuration and collection initialize before resuming a due check.
	// This grace period must not become a new full scheduling interval.
	initialTimer := time.NewTimer(initialPatrolStartDelay)
	defer initialTimer.Stop()

	select {
	case <-initialTimer.C:
	case <-p.stopCh:
		if !initialTimer.Stop() {
			select {
			case <-initialTimer.C:
			default:
			}
		}
		return
	case <-ctx.Done():
		if !initialTimer.Stop() {
			select {
			case <-initialTimer.C:
			default:
			}
		}
		return
	}

	p.mu.RLock()
	interval := p.config.GetInterval()
	configCh := p.configChanged
	p.mu.RUnlock()

	next, trigger := p.initialPatrolSchedule(time.Now(), interval)
	if trigger != TriggerReasonStartup {
		p.setNextPatrolSchedule(next, interval)
	}
	timer := time.NewTimer(time.Until(next))
	defer timer.Stop()

	for {
		select {
		case <-timer.C:
			if trigger == TriggerReasonStartup {
				p.runPatrolWithTrigger(ctx, trigger, nil)
				// Keep the first-install cadence anchored after its startup check.
				next = time.Now().Add(interval)
			} else {
				// Persist the next deadline before starting work. Collapse missed
				// intervals into one due check rather than replaying a backlog.
				next = nextPatrolDeadline(next, time.Now(), interval)
				p.setNextPatrolSchedule(next, interval)
				p.runPatrolWithTrigger(ctx, trigger, nil)
			}
			trigger = TriggerReasonScheduled
			p.setNextPatrolSchedule(next, interval)
			timer.Reset(time.Until(next))

		case alert := <-p.adHocTrigger:
			// Run immediate targeted patrol for this alert
			log.Info().Str("alert_identifier", alert.ID).Msg("patrol triggered by alert")
			p.runTargetedPatrol(ctx, alert)

		case <-configCh:
			// An operator interval change deliberately resets the timer. Persist
			// that exact reset too, even if no new full run has completed yet.
			p.mu.RLock()
			newInterval := p.config.GetInterval()
			p.mu.RUnlock()

			if newInterval != interval {
				interval = newInterval
				if trigger != TriggerReasonStartup {
					next = time.Now().Add(interval)
					p.setNextPatrolSchedule(next, interval)
					timer.Reset(time.Until(next))
				}
				log.Info().
					Dur("interval", interval).
					Msg("Patrol schedule reset to new interval")
			}

		case <-p.stopCh:
			return

		case <-ctx.Done():
			return
		}
	}
}

// remindSweepLoop fires SweepWillFixLaterReminders on a 1h cadence so
// will_fix_later commitments past their RemindAt deadline surface even
// when patrol detection is sparse or the operational issue transiently
// clears between runs. Runs an initial sweep ~30s after start so a
// restart-with-overdue does not have to wait a full hour.
func (p *PatrolService) remindSweepLoop(ctx context.Context) {
	sweep := func() {
		if p.findings == nil {
			return
		}
		swept := p.findings.SweepWillFixLaterReminders(time.Now())
		if swept > 0 {
			log.Info().Int("swept", swept).Msg("Patrol remind-sweep promoted overdue will_fix_later commitments")
		}
	}

	initialTimer := time.NewTimer(remindSweepInitialDelay)
	defer initialTimer.Stop()
	select {
	case <-initialTimer.C:
		sweep()
	case <-p.stopCh:
		return
	case <-ctx.Done():
		return
	}

	ticker := time.NewTicker(remindSweepInterval)
	defer ticker.Stop()
	for {
		select {
		case <-ticker.C:
			sweep()
		case <-p.stopCh:
			return
		case <-ctx.Done():
			return
		}
	}
}

func (p *PatrolService) initialPatrolSchedule(now time.Time, interval time.Duration) (time.Time, TriggerReason) {
	if p.runHistoryStore != nil {
		if next, savedInterval := p.runHistoryStore.Schedule(); !next.IsZero() && savedInterval == interval {
			return next, TriggerReasonScheduled
		}
	}
	// Upgrade old history without a timer cursor. Use all retained full runs:
	// recent scoped/verification activity must not hide or move the cadence.
	if last := p.LastSuccessfulFullPatrolAt(); !last.IsZero() {
		return last.Add(interval), TriggerReasonScheduled
	}
	return now, TriggerReasonStartup
}

func nextPatrolDeadline(due, now time.Time, interval time.Duration) time.Time {
	next := due.Add(interval)
	if !next.After(now) {
		return now.Add(interval)
	}
	return next
}

func (p *PatrolService) setNextPatrolSchedule(next time.Time, interval time.Duration) {
	p.mu.Lock()
	p.nextScheduledAt = next
	enabled, store := p.config.Enabled, p.runHistoryStore
	p.mu.Unlock()
	// Disabled timer ticks are not checks. In particular, first-time provider
	// setup must not leave a durable cursor that suppresses the startup check.
	if enabled && store != nil {
		if err := store.SetSchedule(next, interval); err != nil {
			log.Error().Err(err).Time("next_scheduled_at", next).Msg("Failed to persist Patrol schedule")
		}
	}
}

// LastSuccessfulFullPatrolAt returns the completion time of the most recent
// full patrol run that finished without errors, or the zero time when none
// exists. The manual-run cadence gate uses this rather than lastFullPatrol so
// a failed run does not consume the Community hourly slot.
func (p *PatrolService) LastSuccessfulFullPatrolAt() time.Time {
	if p == nil || p.runHistoryStore == nil {
		return time.Time{}
	}
	var last time.Time
	for _, run := range p.runHistoryStore.GetAll() {
		if run.CompletedAt.IsZero() || !isSuccessfulFullPatrolRun(run) {
			continue
		}
		if run.CompletedAt.After(last) {
			last = run.CompletedAt
		}
	}
	return last
}

func patrolRecencyFromHistory(runHistory []PatrolRunRecord) (time.Time, time.Time) {
	var lastActivity time.Time
	var lastFullPatrol time.Time
	for _, run := range runHistory {
		if run.CompletedAt.IsZero() {
			continue
		}
		if lastActivity.IsZero() || run.CompletedAt.After(lastActivity) {
			lastActivity = run.CompletedAt
		}
		if isFullPatrolRun(run) && (lastFullPatrol.IsZero() || run.CompletedAt.After(lastFullPatrol)) {
			lastFullPatrol = run.CompletedAt
		}
	}
	return lastActivity, lastFullPatrol
}

// runPatrol executes a scheduled patrol run
func (p *PatrolService) runPatrol(ctx context.Context) {
	p.runPatrolWithTrigger(ctx, TriggerReasonScheduled, nil)
}

type patrolRunStart struct {
	id        string
	startedAt time.Time
}

func (p *PatrolService) recordAcceptedRunFailure(start *patrolRunStart, trigger TriggerReason, summary, detail string) {
	if start == nil || p.runHistoryStore == nil {
		return
	}
	completedAt := time.Now()
	duration := completedAt.Sub(start.startedAt)
	p.runHistoryStore.Add(PatrolRunRecord{
		ID:              start.id,
		StartedAt:       start.startedAt,
		CompletedAt:     completedAt,
		Duration:        duration,
		DurationMs:      duration.Milliseconds(),
		Type:            "patrol",
		TriggerReason:   string(trigger),
		FindingsSummary: "Analysis incomplete (1 error)",
		FindingIDs:      []string{},
		ErrorCount:      1,
		Status:          "error",
		ErrorSummary:    summary,
		ErrorDetail:     detail,
	})
	p.mu.Lock()
	p.lastActivity = completedAt
	p.lastFullPatrol = completedAt
	p.lastDuration = duration
	p.resourcesChecked = 0
	p.errorCount = 1
	p.mu.Unlock()
}

// runPatrolWithTrigger executes a patrol run with trigger context
func (p *PatrolService) runPatrolWithTrigger(ctx context.Context, trigger TriggerReason, scope *PatrolScope) {
	p.runPatrolWithTriggerStart(ctx, trigger, scope, nil)
}

// runPatrolWithTriggerStart executes a patrol run, optionally using a start
// reservation that was accepted synchronously by the manual-run API. Keeping
// acceptance and execution on the same reservation removes the interval where
// the API has returned success but status still reports no active run.
func (p *PatrolService) runPatrolWithTriggerStart(ctx context.Context, trigger TriggerReason, scope *PatrolScope, acceptedStart *patrolRunStart) {
	p.mu.RLock()
	cfg := p.config
	breaker := p.circuitBreaker
	p.mu.RUnlock()

	// Demo/mock mode simulates patrol passes without a provider. Checked live
	// (not from the boot-time config snapshot) because release demo instances
	// enable mock fixtures only after the license sync authorizes them.
	if IsDemoMode() {
		if !p.runDemoPatrolCycleWithStart(trigger, acceptedStart) && acceptedStart != nil {
			p.recordAcceptedRunFailure(acceptedStart, trigger,
				"Patrol demo state unavailable",
				"Pulse accepted the run, but demo infrastructure state was not available.")
		}
		return
	}

	if !cfg.Enabled {
		if acceptedStart != nil {
			p.recordAcceptedRunFailure(acceptedStart, trigger,
				"Patrol disabled before execution",
				"Pulse accepted the run, but Patrol was disabled before execution began.")
			p.endRun()
		}
		return
	}
	if reason := strings.TrimSpace(cfg.RuntimeBlockedReason); reason != "" {
		if acceptedStart != nil {
			p.recordAcceptedRunFailure(acceptedStart, trigger,
				"Patrol runtime became unavailable",
				reason)
			p.endRun()
		}
		p.setBlockedReasonWithCause(reason, cfg.RuntimeBlockedCause)
		p.raiseBlockedRunFinding(reason, cfg.RuntimeBlockedCause)
		log.Info().Str("reason", reason).Str("cause", string(cfg.RuntimeBlockedCause)).Msg("AI Patrol: Skipping run - runtime readiness blocked")
		return
	}

	runStart := acceptedStart
	if runStart == nil {
		var accepted bool
		runStart, accepted = p.beginRun("full")
		if !accepted {
			return
		}
	}
	defer p.endRun()

	start := runStart.startedAt
	runID := runStart.id
	executionID := uuid.NewString()
	patrolType := "patrol"
	GetPatrolMetrics().RecordRun(string(trigger), "full")

	log.Debug().Msg("AI Patrol: Starting patrol run")

	// Track run statistics
	var runStats struct {
		resourceCount     int
		nodesChecked      int
		guestsChecked     int
		dockerChecked     int
		storageChecked    int
		hostsChecked      int
		trueNASChecked    int
		pbsChecked        int
		pmgChecked        int
		kubernetesChecked int
		newFindings       int
		existingFindings  int
		rejectedFindings  int
		triageFlags       int
		triageSkippedLLM  bool
		findingIDs        []string
		errors            int
		lastAIError       error // Preserve original error for circuit breaker categorization
		errorSummary      string
		errorDetail       string
		aiAnalysis        *AIAnalysisResult // Stores the AI's analysis for the run record
		runtimeResolved   bool
	}
	var newFindings []*Finding

	// Get current state
	if !p.hasPatrolRuntimeInputs() {
		log.Warn().Msg("AI Patrol: No runtime state available")
		if acceptedStart != nil {
			p.recordAcceptedRunFailure(runStart, trigger,
				"Patrol runtime state unavailable",
				"Pulse accepted the run, but no infrastructure state was available for analysis.")
		}
		return
	}

	state := p.currentPatrolRuntimeState()

	// Helper to track findings
	// Note: Only warning+ severity findings count toward newFindings since watch/info are filtered from UI
	trackFinding := func(f *Finding) bool {
		isNew := p.recordFinding(f)
		if isNew {
			// Only count warning+ findings as "new" for user-facing stats
			if f.Severity == FindingSeverityWarning || f.Severity == FindingSeverityCritical {
				runStats.newFindings++
				newFindings = append(newFindings, f)
			}
		} else {
			runStats.existingFindings++
		}

		// Only track warning+ severity finding IDs in the run record
		if f.Severity == FindingSeverityWarning || f.Severity == FindingSeverityCritical {
			runStats.findingIDs = append(runStats.findingIDs, f.ID)
		}

		return isNew
	}

	// Count resources for statistics from the patrol runtime state so scoped and
	// unscoped runs share the same read-state-first semantics.
	resourceCounts := patrolRuntimeCountResources(state)
	if cfg.AnalyzeNodes {
		runStats.nodesChecked = resourceCounts.nodes
	}
	if cfg.AnalyzeGuests {
		runStats.guestsChecked = resourceCounts.guests
	}
	if cfg.AnalyzeDocker {
		runStats.dockerChecked = resourceCounts.docker
	}
	if cfg.AnalyzeStorage {
		runStats.storageChecked = resourceCounts.storage
	}
	if cfg.AnalyzePBS {
		runStats.pbsChecked = resourceCounts.pbs
	}
	if cfg.AnalyzePMG {
		runStats.pmgChecked = resourceCounts.pmg
	}
	if cfg.AnalyzeHosts {
		runStats.hostsChecked = resourceCounts.hosts
		runStats.trueNASChecked = resourceCounts.truenas
	}
	if cfg.AnalyzeKubernetes {
		runStats.kubernetesChecked = resourceCounts.kubernetes
	}
	runStats.resourceCount = runStats.nodesChecked + runStats.guestsChecked +
		runStats.dockerChecked + runStats.storageChecked + runStats.pbsChecked + runStats.pmgChecked + runStats.hostsChecked +
		runStats.trueNASChecked +
		runStats.kubernetesChecked

	// Run update-safety watcher on every patrol cycle regardless of LLM availability.
	// Direct FindingsStore.Add bypasses push-notification (MVP: not an emergency).
	if cfg.AnalyzeDocker && p.updateSafetyWatcher != nil {
		updateEmit, updateResolve := p.updateSafetyWatcher.Observe(patrolUpdateSafetyDockerHosts(state), time.Now())
		for _, f := range updateEmit {
			if !p.findings.Add(f) {
				log.Debug().Str("finding_id", f.ID).Msg("AI Patrol: update-safety finding already in store")
			}
		}
		for _, s := range updateResolve {
			p.findings.ResolveWithReason(s.DedupKey, s.Reason)
		}
	}

	if cfg.AnalyzeHosts && p.aptWorkflowWatcher != nil {
		aptEmit, aptResolve := p.aptWorkflowWatcher.Observe(state, p.findings.GetActive(FindingSeverityInfo), time.Now())
		for _, f := range aptEmit {
			trackFinding(f)
		}
		for _, sentinel := range aptResolve {
			if p.findings.ResolveWithReason(sentinel.DedupKey, sentinel.Reason) {
				p.mu.RLock()
				resolveUnified := p.unifiedFindingResolver
				p.mu.RUnlock()
				if resolveUnified != nil {
					resolveUnified(sentinel.DedupKey)
				}
			}
		}
	}

	if cfg.AnalyzeGuests && p.proxmoxGuestLifecycleWatcher != nil {
		proxmoxEmit, proxmoxResolve := p.proxmoxGuestLifecycleWatcher.Observe(state, p.findings.GetActive(FindingSeverityInfo), time.Now())
		for _, f := range proxmoxEmit {
			trackFinding(f)
		}
		for _, sentinel := range proxmoxResolve {
			if p.findings.ResolveWithReason(sentinel.DedupKey, sentinel.Reason) {
				p.mu.RLock()
				resolveUnified := p.unifiedFindingResolver
				p.mu.RUnlock()
				if resolveUnified != nil {
					resolveUnified(sentinel.DedupKey)
				}
			}
		}
	}

	// Run PDM alert bridge on every patrol cycle if a source is configured.
	// Nil source guard mirrors the MVP no-op behavior; the real HTTP client
	// is wired in a follow-on. Direct FindingsStore.Add bypasses push.
	if p.pdmAlertBridge != nil && p.pdmAlertBridge.source != nil {
		pdmEmit, pdmResolve := p.pdmAlertBridge.Observe(ctx, time.Now())
		for _, f := range pdmEmit {
			if !p.findings.Add(f) {
				log.Debug().Str("finding_id", f.ID).Msg("AI Patrol: PDM alert finding already in store")
			}
		}
		for _, s := range pdmResolve {
			p.findings.ResolveWithReason(s.DedupKey, s.Reason)
		}
	}

	// Acquire a breaker probe only when this run is actually ready to call the
	// provider. Early collection/scope exits must not strand a half-open probe.
	aiServiceEnabled := p.retryProviderInitIfNeeded(ctx)
	llmAllowed := true
	if aiServiceEnabled && breaker != nil {
		llmAllowed = breaker.Allow()
	}
	if !llmAllowed {
		log.Warn().Msg("AI Patrol: Circuit breaker is open (LLM calls blocked)")
	}
	canRunLLM := aiServiceEnabled && llmAllowed

	// Check if we can run LLM analysis (AI-only patrol)
	if !canRunLLM {
		reason, cause := p.providerUnavailableReason()
		if aiServiceEnabled && !llmAllowed {
			reason = "circuit breaker is open"
			cause = PatrolFailureCauseCircuitOpen
			GetPatrolMetrics().RecordCircuitBlock()
		} else {
			// A run skipped for a persistent provider state (never attempted)
			// surfaces as the deduped runtime finding; a transiently open
			// breaker does not, because the attempts that opened it already
			// raised their own specific finding.
			trackFinding(newPatrolRuntimeFailureFinding(patrolBlockedRunFailure(reason, cause), time.Now()))
		}
		p.setBlockedReasonWithCause(reason, cause)
		log.Info().Str("reason", reason).Str("cause", string(cause)).Msg("AI Patrol: Skipping run - AI unavailable")
		runStats.errors++
		runStats.errorSummary = "Patrol provider unavailable"
		runStats.errorDetail = reason
	} else {
		p.clearBlockedReason()
		// Ensure stream state is clean for this run before the first streamed event.
		p.resetStreamForRun(runID)
		// Run agentic AI analysis — the LLM uses tools to investigate and reports findings
		aiResult, aiErr := p.runAIAnalysisState(ctx, state, scope, executionID)
		if aiResult != nil {
			runStats.aiAnalysis = aiResult
			runStats.rejectedFindings = aiResult.RejectedFindings
			runStats.triageFlags = aiResult.TriageFlags
			runStats.triageSkippedLLM = aiResult.TriageSkippedLLM

			// Findings are recorded by the Patrol finding tools before the model
			// response completes. Preserve their run-level accounting even when a
			// later provider error interrupts the same analysis.
			for _, f := range aiResult.Findings {
				if f.Severity == FindingSeverityWarning || f.Severity == FindingSeverityCritical {
					runStats.findingIDs = append(runStats.findingIDs, f.ID)
					stored := p.findings.Get(f.ID)
					if stored != nil && stored.TimesRaised <= 1 {
						runStats.newFindings++
						newFindings = append(newFindings, f)
					} else {
						runStats.existingFindings++
					}
				}
			}
		}
		if aiErr != nil {
			log.Warn().Err(aiErr).Msg("AI Patrol: LLM analysis failed")
			runStats.errors++
			runStats.lastAIError = aiErr
			failure := patrolRuntimeFailureFromError(aiErr)
			runStats.errorSummary = failure.Summary
			runStats.errorDetail = failure.Detail
			p.blockOnExhaustedBudget(aiErr, failure)

			// Create a finding to surface this error to the user
			errorFinding := newPatrolRuntimeFailureFinding(failure, time.Now())
			trackFinding(errorFinding)
		} else if aiResult != nil {
			if missing := patrolMissingAssessmentIDs(aiResult); len(missing) > 0 {
				runStats.errors++
				runStats.errorSummary = "Patrol finding assessment incomplete"
				runStats.errorDetail = fmt.Sprintf("No explicit verdict was accepted for active finding IDs: %s", strings.Join(missing, ", "))
			}

			if !aiResult.TriageSkippedLLM {
				runStats.runtimeResolved = p.resolvePatrolRuntimeFailureFinding("full_patrol_success")
			}

			// Attach the deterministic capacity forecasts computed this run to
			// matching findings, so the surface shows a first-class urgency
			// signal (trend/eta) instead of relying on model prose.
			p.stampCapacityForecasts(aiResult.Forecasts)
		}
	}

	// Fold findings that restate an active alert under that alert. Runs after
	// both the deterministic watchers and the model have written this cycle's
	// findings, against the unscoped alert set, so scoped runs cannot release
	// mirrors outside their scope.
	if mirrored := p.reconcileAlertMirrors(); mirrored > 0 {
		log.Debug().Int("changed", mirrored).Msg("AI Patrol: Updated alert-mirror stamps on findings")
	}

	// Count resolved findings: LLM-resolved (via tool) + auto-reconciled stale findings.
	var resolvedCount int
	if runStats.aiAnalysis != nil {
		resolvedCount = len(runStats.aiAnalysis.ResolvedIDs)
		if runStats.runtimeResolved {
			resolvedCount++
		}

		// Auto-resolve stale findings: active findings that were presented to the LLM
		// in seed context but were neither re-reported nor explicitly resolved.
		// Only runs after successful full patrols (not scoped).
		observedIDs := append(append([]string{}, runStats.aiAnalysis.ReportedIDs...), assessmentFindingIDs(runStats.aiAnalysis.Assessments)...)
		autoResolved := p.reconcileStaleFindings(
			observedIDs,
			runStats.aiAnalysis.ResolvedIDs,
			runStats.aiAnalysis.SeededFindingIDs,
			runStats.errors > 0,
		)
		resolvedCount += autoResolved
		if autoResolved > 0 {
			log.Info().
				Int("auto_resolved", autoResolved).
				Msg("AI Patrol: Auto-resolved stale findings after full patrol")
		}
	}

	// Cleanup old resolved findings (always runs, doesn't require LLM)
	cleaned := p.findings.Cleanup(findingCleanupAge)
	if cleaned > 0 {
		log.Debug().Int("cleaned", cleaned).Msg("AI Patrol: Cleaned up old findings")
	}

	// Recover investigations stuck in "running" state (goroutine panicked or was killed)
	p.recoverStuckInvestigations()

	// Retry investigations that failed due to timeout (shorter cooldown than permanent failures)
	p.retryTimedOutInvestigations()

	// AI-based alert review: check active alerts against current state and auto-resolve fixed issues
	// Pass llmAllowed so it knows whether AI calls are allowed.
	alertsResolved := p.reviewAndResolveAlertsState(ctx, state, llmAllowed, executionID)
	if alertsResolved > 0 {
		log.Info().Int("alerts_resolved", alertsResolved).Msg("AI Patrol: Auto-resolved alerts where issues are fixed")
	}

	duration := time.Since(start)
	completedAt := time.Now()

	// Build findings summary string.
	summary := p.findings.GetSummary()
	var findingsSummaryStr string
	var status string
	// Only count critical and warning as active issues (watch/info are filtered from UI)
	totalActive := summary.Critical + summary.Warning
	if totalActive == 0 {
		findingsSummaryStr = "All healthy"
		status = "healthy"
	} else {
		parts := []string{}
		if summary.Critical > 0 {
			parts = append(parts, fmt.Sprintf("%d critical", summary.Critical))
		}
		if summary.Warning > 0 {
			parts = append(parts, fmt.Sprintf("%d warning", summary.Warning))
		}
		findingsSummaryStr = fmt.Sprintf("%s", joinParts(parts))
		if summary.Critical > 0 {
			status = "critical"
		} else {
			status = "issues_found"
		}
	}
	if runStats.errors > 0 {
		status = "error"
		// Don't claim "All healthy" if there were errors - the patrol didn't complete properly
		if findingsSummaryStr == "All healthy" {
			findingsSummaryStr = fmt.Sprintf("Analysis incomplete (%d errors)", runStats.errors)
		}
	}

	// Create run record
	runRecord := PatrolRunRecord{
		ID:                runID,
		StartedAt:         start,
		CompletedAt:       completedAt,
		Duration:          duration,
		DurationMs:        duration.Milliseconds(),
		Type:              patrolType,
		TriggerReason:     string(trigger),
		ResourcesChecked:  runStats.resourceCount,
		NodesChecked:      runStats.nodesChecked,
		GuestsChecked:     runStats.guestsChecked,
		DockerChecked:     runStats.dockerChecked,
		StorageChecked:    runStats.storageChecked,
		HostsChecked:      runStats.hostsChecked,
		TrueNASChecked:    runStats.trueNASChecked,
		PBSChecked:        runStats.pbsChecked,
		PMGChecked:        runStats.pmgChecked,
		KubernetesChecked: runStats.kubernetesChecked,
		NewFindings:       runStats.newFindings,
		ExistingFindings:  runStats.existingFindings,
		RejectedFindings:  runStats.rejectedFindings,
		ResolvedFindings:  resolvedCount,
		FindingsSummary:   findingsSummaryStr,
		FindingIDs:        patrolRunFindingIDs(runStats.findingIDs, assessmentsForRun(runStats.aiAnalysis)),
		ErrorCount:        runStats.errors,
		Status:            status,
		ErrorSummary:      runStats.errorSummary,
		ErrorDetail:       runStats.errorDetail,
	}

	if scope != nil {
		runRecord.ScopeResourceIDs = scope.ResourceIDs
		runRecord.ScopeResourceTypes = scope.ResourceTypes
		runRecord.ScopeContext = scope.Context
		runRecord.ObjectiveContext = clonePatrolObjectiveContext(scope.ObjectiveContext)
		runRecord.AlertIdentifier = scope.AlertIdentifier
		runRecord.FindingID = scope.FindingID
	}

	// Add AI analysis details if available
	if runStats.aiAnalysis != nil {
		runRecord.AIAnalysis = patrolRunAIAnalysisForRecord(runStats.aiAnalysis, patrolRunAnalysisRecordContext{
			ResourcesChecked:  runStats.resourceCount,
			NewFindings:       runStats.newFindings,
			ExistingFindings:  runStats.existingFindings,
			ResolvedFindings:  resolvedCount,
			UncertainFindings: patrolUncertainAssessmentCount(runStats.aiAnalysis.Assessments),
			ErrorCount:        runStats.errors,
		})
		runRecord.FindingAssessments = append([]PatrolFindingAssessment(nil), runStats.aiAnalysis.Assessments...)
		runRecord.InputTokens = runStats.aiAnalysis.InputTokens
		runRecord.OutputTokens = runStats.aiAnalysis.OutputTokens
		runRecord.CacheCreationInputTokens = runStats.aiAnalysis.CacheCreationInputTokens
		runRecord.CacheReadInputTokens = runStats.aiAnalysis.CacheReadInputTokens
		runRecord.TriageFlags = runStats.triageFlags
		runRecord.TriageSkippedLLM = runStats.triageSkippedLLM
		toolCalls := runStats.aiAnalysis.ToolCalls
		if len(toolCalls) > MaxToolCallsPerRun {
			toolCalls = toolCalls[:MaxToolCallsPerRun]
		}
		runRecord.ToolCalls = toolCalls
		runRecord.ToolCallCount = len(runStats.aiAnalysis.ToolCalls)
		log.Debug().
			Int("response_length", len(runStats.aiAnalysis.Response)).
			Int("input_tokens", runStats.aiAnalysis.InputTokens).
			Int("output_tokens", runStats.aiAnalysis.OutputTokens).
			Int("tool_calls", runRecord.ToolCallCount).
			Msg("AI Patrol: Storing AI analysis in run record")
	} else {
		log.Debug().Msg("AI Patrol: No AI analysis to store (aiAnalysis is nil)")
	}

	p.mu.Lock()
	p.lastActivity = completedAt
	p.lastFullPatrol = completedAt
	p.lastDuration = duration
	p.resourcesChecked = runStats.resourceCount
	p.errorCount = runStats.errors
	p.mu.Unlock()

	// Record circuit breaker result only if we actually attempted LLM calls.
	// canRunLLM is true only when AI is enabled, licensed, AND breaker allowed.
	// Use error categorization so non-transient errors (auth failures, insufficient
	// credits) don't trip the breaker — those won't be fixed by waiting.
	recordPatrolCircuitResult(breaker, canRunLLM, runStats.errors, runStats.lastAIError)

	// Add to history store (handles persistence automatically)
	p.runHistoryStore.Add(runRecord)

	log.Info().
		Str("type", patrolType).
		Dur("duration", duration).
		Int("resources", runStats.resourceCount).
		Int("new_findings", runStats.newFindings).
		Int("resolved", resolvedCount).
		Int("critical", summary.Critical).
		Int("warning", summary.Warning).
		Int("watch", summary.Watch).
		Msg("AI Patrol: Completed patrol run")

	p.dispatchPatrolInvestigations(runStats.aiAnalysis)
}

func patrolFindingSummaryForRun(findings []*Finding, findingIDs []string) FindingsSummary {
	owned := make(map[string]bool, len(findingIDs))
	for _, findingID := range findingIDs {
		if findingID = strings.TrimSpace(findingID); findingID != "" {
			owned[findingID] = true
		}
	}
	var summary FindingsSummary
	for _, finding := range findings {
		if finding == nil || !owned[finding.ID] {
			continue
		}
		summary.Total++
		switch finding.Severity {
		case FindingSeverityCritical:
			summary.Critical++
		case FindingSeverityWarning:
			summary.Warning++
		case FindingSeverityWatch:
			summary.Watch++
		case FindingSeverityInfo:
			summary.Info++
		}
	}
	return summary
}

// runScopedPatrol runs a patrol on a filtered subset of resources.
// This provides token-efficient analysis for event-driven patrols.
func (p *PatrolService) runScopedPatrol(ctx context.Context, scope PatrolScope) {
	p.runScopedPatrolWithStart(ctx, scope, nil, true)
}

// runScopedPatrolWithStart executes a scoped run either from the background
// trigger queue or from a reservation accepted synchronously by the manual-run
// API. Background triggers retain their bounded retry behaviour. A manual
// reservation is already durable and therefore must either produce a run
// record or an explicit terminal failure; it is never silently re-queued.
func (p *PatrolService) runScopedPatrolWithStart(ctx context.Context, scope PatrolScope, acceptedStart *patrolRunStart, retryOnBusy bool) {
	p.mu.RLock()
	cfg := p.config
	breaker := p.circuitBreaker
	p.mu.RUnlock()

	// Demo instances simulate scheduled patrol passes only; event-driven
	// scoped runs would hit the real provider path.
	if IsDemoMode() {
		if acceptedStart != nil {
			p.recordScopedPatrolScopeFailure(acceptedStart.startedAt, acceptedStart.id, scope,
				PatrolScopeResolution{RequestedResourceIDs: append([]string(nil), scope.ResourceIDs...)},
				"Scoped Patrol is unavailable in demo mode",
				"Pulse accepted the scoped run, but the demo runtime cannot execute event-driven scoped checks.")
			p.endRun()
		}
		return
	}

	if !cfg.Enabled {
		if acceptedStart != nil {
			p.recordScopedPatrolScopeFailure(acceptedStart.startedAt, acceptedStart.id, scope,
				PatrolScopeResolution{RequestedResourceIDs: append([]string(nil), scope.ResourceIDs...)},
				"Patrol disabled before scoped execution",
				"Pulse accepted the scoped run, but Patrol was disabled before execution began.")
			p.endRun()
		}
		return
	}
	if reason := strings.TrimSpace(cfg.RuntimeBlockedReason); reason != "" {
		if acceptedStart != nil {
			p.recordScopedPatrolScopeFailure(acceptedStart.startedAt, acceptedStart.id, scope,
				PatrolScopeResolution{RequestedResourceIDs: append([]string(nil), scope.ResourceIDs...)},
				"Patrol runtime became unavailable",
				reason)
			p.endRun()
		}
		p.setBlockedReasonWithCause(reason, cfg.RuntimeBlockedCause)
		log.Info().Str("reason", reason).Str("cause", string(cfg.RuntimeBlockedCause)).Msg("AI Patrol: Skipping scoped run - runtime readiness blocked")
		return
	}

	runStart := acceptedStart
	if runStart == nil {
		var accepted bool
		runStart, accepted = p.beginRun("scoped")
		// Re-queue background triggers with backoff if retries remain.
		if !accepted && retryOnBusy && scope.RetryCount < scopedPatrolMaxRetries {
			scope.RetryCount++
			backoff := scopedPatrolRetryBackoff1
			if scope.RetryCount == scopedPatrolMaxRetries {
				backoff = scopedPatrolRetryBackoff2
			}
			scope.RetryAfter = time.Now().Add(backoff)
			if tm := p.GetTriggerManager(); tm != nil {
				tm.TriggerPatrol(scope)
				log.Info().
					Int("retry", scope.RetryCount).
					Dur("backoff", backoff).
					Strs("resources", scope.ResourceIDs).
					Msg("AI Patrol: Re-queued dropped scoped patrol with backoff")
			}
		} else if !accepted && retryOnBusy {
			GetPatrolMetrics().RecordScopedDroppedFinal()
			log.Error().
				Strs("resources", scope.ResourceIDs).
				Str("reason", string(scope.Reason)).
				Msg("AI Patrol: Scoped patrol permanently dropped after 2 retries")
		}
		if !accepted {
			return
		}
	}
	defer p.endRun()

	start := runStart.startedAt
	runID := runStart.id
	executionID := uuid.NewString()
	if scope.Reason == TriggerReasonObjectiveChanged && !p.beginObjectivePlanning(scope, start) {
		return
	}
	GetPatrolMetrics().RecordRun(string(scope.Reason), "scoped")
	var runStats struct {
		resourceCount     int
		nodesChecked      int
		guestsChecked     int
		dockerChecked     int
		storageChecked    int
		hostsChecked      int
		trueNASChecked    int
		pbsChecked        int
		pmgChecked        int
		kubernetesChecked int
		newFindings       int
		existingFindings  int
		rejectedFindings  int
		triageFlags       int
		triageSkippedLLM  bool
		findingIDs        []string
		errors            int
		lastAIError       error
		errorSummary      string
		errorDetail       string
		aiAnalysis        *AIAnalysisResult
		runtimeResolved   bool
	}

	// Get current state
	if !p.hasPatrolRuntimeInputs() {
		log.Warn().Msg("AI Patrol: No runtime state available for scoped patrol")
		p.recordScopedPatrolScopeFailure(start, runID, scope, PatrolScopeResolution{RequestedResourceIDs: append([]string(nil), scope.ResourceIDs...)}, "Patrol runtime state unavailable", "No normal collection state was available for the requested scoped Patrol run.")
		return
	}

	fullState := p.currentPatrolRuntimeState()
	requestedScope := scope
	resolvedScope, scopeResolution := resolvePatrolScopeState(fullState, scope)
	if len(scopeResolution.UnmatchedResourceIDs) > 0 || len(scopeResolution.AmbiguousResourceIDs) > 0 ||
		(len(scopeResolution.RequestedResourceIDs) > 0 && len(scopeResolution.ResolvedResourceIDs) == 0) {
		p.recordScopedPatrolScopeFailure(start, runID, requestedScope, scopeResolution,
			"Patrol scope could not be resolved",
			"One or more requested resource identities did not resolve exactly to the current Patrol collection state.")
		return
	}
	scope = resolvedScope

	// Filter state based on scope
	filteredState := p.filterStateByScopeState(fullState, scope)
	effectiveScopeIDs := patrolRuntimeSortedResourceIDs(filteredState)

	resourceCounts := patrolRuntimeCountResources(filteredState)
	resourceCount := 0
	if cfg.AnalyzeNodes {
		resourceCount += resourceCounts.nodes
	}
	if cfg.AnalyzeGuests {
		resourceCount += resourceCounts.guests
	}
	if cfg.AnalyzeDocker {
		resourceCount += resourceCounts.docker
	}
	if cfg.AnalyzeStorage {
		resourceCount += resourceCounts.storage
	}
	if cfg.AnalyzePBS {
		resourceCount += resourceCounts.pbs
	}
	if cfg.AnalyzeHosts {
		resourceCount += resourceCounts.hosts
		resourceCount += resourceCounts.truenas
	}
	if cfg.AnalyzeKubernetes {
		resourceCount += resourceCounts.kubernetes
	}
	if cfg.AnalyzePMG {
		resourceCount += resourceCounts.pmg
	}
	// Canonical resource kinds without a legacy snapshot projection are still
	// real scoped work. They are admitted only through exact unified-resource
	// matching and remain available to the objective seed and governed tools.
	resourceCount += resourceCounts.unified

	if resourceCount == 0 {
		log.Debug().
			Strs("requested_ids", scope.ResourceIDs).
			Strs("requested_types", scope.ResourceTypes).
			Int("effective_scope_count", len(effectiveScopeIDs)).
			Msg("AI Patrol: No resources matched scope filter")
		p.recordScopedPatrolScopeFailure(start, runID, scope, scopeResolution, "No resources matched requested Patrol scope", "The requested resource identities did not resolve to any enabled resource in the current normal Patrol collection state.")
		return
	}

	log.Debug().
		Strs("requested_ids", scope.ResourceIDs).
		Strs("requested_types", scope.ResourceTypes).
		Strs("effective_scope_ids", patrolLogResourceIDs(effectiveScopeIDs)).
		Int("effective_scope_count", len(effectiveScopeIDs)).
		Int("resource_count", resourceCount).
		Str("reason", string(scope.Reason)).
		Msg("AI Patrol: Running scoped analysis")

	// Track run statistics
	if cfg.AnalyzeNodes {
		runStats.nodesChecked = resourceCounts.nodes
	}
	if cfg.AnalyzeGuests {
		runStats.guestsChecked = resourceCounts.guests
	}
	if cfg.AnalyzeDocker {
		runStats.dockerChecked = resourceCounts.docker
	}
	if cfg.AnalyzeStorage {
		runStats.storageChecked = resourceCounts.storage
	}
	if cfg.AnalyzePBS {
		runStats.pbsChecked = resourceCounts.pbs
	}
	if cfg.AnalyzeHosts {
		runStats.hostsChecked = resourceCounts.hosts
		runStats.trueNASChecked = resourceCounts.truenas
	}
	if cfg.AnalyzeKubernetes {
		runStats.kubernetesChecked = resourceCounts.kubernetes
	}
	if cfg.AnalyzePMG {
		runStats.pmgChecked = resourceCounts.pmg
	}
	runStats.resourceCount = resourceCount

	// Acquire a breaker probe only after the requested scope has resolved to
	// real work. This keeps collection-only failures outside provider health.
	aiServiceEnabled := p.retryProviderInitIfNeeded(ctx)
	llmAllowed := true
	if aiServiceEnabled && breaker != nil {
		llmAllowed = breaker.Allow()
	}
	if !llmAllowed {
		log.Warn().Msg("AI Patrol: Circuit breaker is open for scoped patrol (LLM calls blocked)")
	}
	canRunLLM := aiServiceEnabled && llmAllowed

	if !canRunLLM {
		reason, cause := p.providerUnavailableReason()
		if aiServiceEnabled && !llmAllowed {
			reason = "circuit breaker is open"
			cause = PatrolFailureCauseCircuitOpen
			GetPatrolMetrics().RecordCircuitBlock()
		} else {
			p.raiseBlockedRunFinding(reason, cause)
		}
		p.setBlockedReasonWithCause(reason, cause)
		log.Info().Str("reason", reason).Str("cause", string(cause)).Msg("AI Patrol: Skipping scoped run - AI unavailable")
		return
	}

	{
		p.clearBlockedReason()
		if !scope.NoStream {
			// Ensure stream state is clean for this run before the first streamed event.
			p.resetStreamForRun(runID)
		}
		// Run agentic AI analysis on filtered state with scope
		aiResult, aiErr := p.runAIAnalysisState(ctx, filteredState, &scope, executionID)
		if aiResult != nil {
			runStats.aiAnalysis = aiResult
			runStats.rejectedFindings = aiResult.RejectedFindings
			runStats.triageFlags = aiResult.TriageFlags
			runStats.triageSkippedLLM = aiResult.TriageSkippedLLM

			// Keep accepted finding evidence attached to this run even if the
			// provider fails after the corresponding tool call completed.
			for _, f := range aiResult.Findings {
				if f.Severity == FindingSeverityWarning || f.Severity == FindingSeverityCritical {
					runStats.findingIDs = append(runStats.findingIDs, f.ID)
					stored := p.findings.Get(f.ID)
					if stored != nil && stored.TimesRaised <= 1 {
						runStats.newFindings++
					} else {
						runStats.existingFindings++
					}
				}
			}
		}
		if aiErr != nil {
			log.Warn().Err(aiErr).Msg("AI Patrol (scoped): LLM analysis failed")
			runStats.errors++
			runStats.lastAIError = aiErr
			failure := patrolRuntimeFailureFromError(aiErr)
			runStats.errorSummary = failure.Summary
			runStats.errorDetail = failure.Detail
			p.blockOnExhaustedBudget(aiErr, failure)
			errorFinding := newPatrolRuntimeFailureFinding(failure, time.Now())
			if p.recordFinding(errorFinding) {
				runStats.newFindings++
			} else {
				runStats.existingFindings++
			}
			runStats.findingIDs = append(runStats.findingIDs, errorFinding.ID)
		} else if aiResult != nil {
			if missing := patrolMissingAssessmentIDs(aiResult); len(missing) > 0 {
				runStats.errors++
				runStats.errorSummary = "Patrol finding assessment incomplete"
				runStats.errorDetail = fmt.Sprintf("No explicit verdict was accepted for active finding IDs: %s", strings.Join(missing, ", "))
			}
			if !aiResult.TriageSkippedLLM {
				runStats.runtimeResolved = p.resolvePatrolRuntimeFailureFinding("scoped_patrol_success")
			}
		}
	}

	duration := time.Since(start)
	completedAt := time.Now()

	// Build a run-owned findings summary. Supporting resources included in the
	// effective scope provide diagnostic context, but an unrelated finding on
	// one of those resources must not make this targeted run appear unhealthy.
	runFindingIDs := patrolRunFindingIDs(runStats.findingIDs, assessmentsForRun(runStats.aiAnalysis))
	summary := patrolFindingSummaryForRun(p.findings.GetActive(FindingSeverityWarning), runFindingIDs)
	var findingsSummaryStr string
	var status string
	totalActive := summary.Critical + summary.Warning
	if totalActive == 0 {
		findingsSummaryStr = "All healthy"
		status = "healthy"
	} else {
		parts := []string{}
		if summary.Critical > 0 {
			parts = append(parts, fmt.Sprintf("%d critical", summary.Critical))
		}
		if summary.Warning > 0 {
			parts = append(parts, fmt.Sprintf("%d warning", summary.Warning))
		}
		findingsSummaryStr = fmt.Sprintf("%s", joinParts(parts))
		if summary.Critical > 0 {
			status = "critical"
		} else {
			status = "issues_found"
		}
	}
	if runStats.errors > 0 {
		status = "error"
		if findingsSummaryStr == "All healthy" {
			findingsSummaryStr = fmt.Sprintf("Analysis incomplete (%d errors)", runStats.errors)
		}
	}
	resolvedFindings := 0
	if runStats.aiAnalysis != nil {
		resolvedFindings = len(runStats.aiAnalysis.ResolvedIDs)
		if runStats.runtimeResolved {
			resolvedFindings++
		}
	}

	runRecord := PatrolRunRecord{
		ID:                        runID,
		StartedAt:                 start,
		CompletedAt:               completedAt,
		Duration:                  duration,
		DurationMs:                duration.Milliseconds(),
		Type:                      "scoped",
		TriggerReason:             string(scope.Reason),
		ScopeResourceIDs:          scopeResolution.RequestedResourceIDs,
		EffectiveScopeResourceIDs: effectiveScopeIDs,
		ScopeResourceTypes:        scope.ResourceTypes,
		ScopeContext:              scope.Context,
		ObjectiveContext:          clonePatrolObjectiveContext(scope.ObjectiveContext),
		AlertIdentifier:           scope.AlertIdentifier,
		FindingID:                 scope.FindingID,
		ResourcesChecked:          runStats.resourceCount,
		NodesChecked:              runStats.nodesChecked,
		GuestsChecked:             runStats.guestsChecked,
		DockerChecked:             runStats.dockerChecked,
		StorageChecked:            runStats.storageChecked,
		HostsChecked:              runStats.hostsChecked,
		TrueNASChecked:            runStats.trueNASChecked,
		PBSChecked:                runStats.pbsChecked,
		PMGChecked:                runStats.pmgChecked,
		KubernetesChecked:         runStats.kubernetesChecked,
		NewFindings:               runStats.newFindings,
		ExistingFindings:          runStats.existingFindings,
		RejectedFindings:          runStats.rejectedFindings,
		ResolvedFindings:          resolvedFindings,
		FindingsSummary:           findingsSummaryStr,
		FindingIDs:                runFindingIDs,
		ErrorCount:                runStats.errors,
		Status:                    status,
		ErrorSummary:              runStats.errorSummary,
		ErrorDetail:               runStats.errorDetail,
	}

	if runStats.aiAnalysis != nil {
		runRecord.AIAnalysis = patrolRunAIAnalysisForRecord(runStats.aiAnalysis, patrolRunAnalysisRecordContext{
			ResourcesChecked:  runStats.resourceCount,
			NewFindings:       runStats.newFindings,
			ExistingFindings:  runStats.existingFindings,
			ResolvedFindings:  resolvedFindings,
			UncertainFindings: patrolUncertainAssessmentCount(runStats.aiAnalysis.Assessments),
			ErrorCount:        runStats.errors,
		})
		runRecord.FindingAssessments = append([]PatrolFindingAssessment(nil), runStats.aiAnalysis.Assessments...)
		runRecord.InputTokens = runStats.aiAnalysis.InputTokens
		runRecord.OutputTokens = runStats.aiAnalysis.OutputTokens
		runRecord.CacheCreationInputTokens = runStats.aiAnalysis.CacheCreationInputTokens
		runRecord.CacheReadInputTokens = runStats.aiAnalysis.CacheReadInputTokens
		runRecord.TriageFlags = runStats.triageFlags
		runRecord.TriageSkippedLLM = runStats.triageSkippedLLM
		toolCalls := runStats.aiAnalysis.ToolCalls
		if len(toolCalls) > MaxToolCallsPerRun {
			toolCalls = toolCalls[:MaxToolCallsPerRun]
		}
		runRecord.ToolCalls = toolCalls
		runRecord.ToolCallCount = len(runStats.aiAnalysis.ToolCalls)
	}

	p.mu.Lock()
	p.lastActivity = completedAt
	p.lastDuration = duration
	p.resourcesChecked = runStats.resourceCount
	p.errorCount = runStats.errors
	p.mu.Unlock()

	recordPatrolCircuitResult(breaker, canRunLLM, runStats.errors, runStats.lastAIError)

	p.runHistoryStore.Add(runRecord)

	log.Info().
		Strs("requested_ids", scopeResolution.RequestedResourceIDs).
		Strs("requested_types", scope.ResourceTypes).
		Strs("effective_scope_ids", patrolLogResourceIDs(effectiveScopeIDs)).
		Int("effective_scope_count", len(effectiveScopeIDs)).
		Dur("duration", duration).
		Int("resources", resourceCount).
		Str("reason", string(scope.Reason)).
		Msg("AI Patrol: Scoped patrol complete")

	p.dispatchPatrolInvestigations(runStats.aiAnalysis)
}

func recordPatrolCircuitResult(breaker *circuit.Breaker, attempted bool, errorCount int, runErr error) {
	if breaker == nil || !attempted {
		return
	}
	if errorCount == 0 {
		breaker.RecordSuccess()
		return
	}
	if runErr == nil {
		runErr = fmt.Errorf("patrol completed with %d errors", errorCount)
	}
	if errors.Is(runErr, ErrCostBudgetExceeded) {
		// Pulse declined to spend; the provider was never called, so there
		// is no provider failure to count (#1789).
		return
	}
	breaker.RecordFailureWithCategory(runErr, circuit.CategorizeError(runErr))
}

func (p *PatrolService) recordScopedPatrolScopeFailure(start time.Time, runID string, scope PatrolScope, resolution PatrolScopeResolution, summary, detail string) {
	completedAt := time.Now()
	record := PatrolRunRecord{
		ID:                        runID,
		StartedAt:                 start,
		CompletedAt:               completedAt,
		Duration:                  completedAt.Sub(start),
		DurationMs:                completedAt.Sub(start).Milliseconds(),
		Type:                      "scoped",
		TriggerReason:             string(scope.Reason),
		ScopeResourceIDs:          append([]string(nil), resolution.RequestedResourceIDs...),
		EffectiveScopeResourceIDs: append([]string(nil), resolution.EffectiveResourceIDs...),
		ScopeResourceTypes:        append([]string(nil), scope.ResourceTypes...),
		ScopeContext:              scope.Context,
		ObjectiveContext:          clonePatrolObjectiveContext(scope.ObjectiveContext),
		AlertIdentifier:           scope.AlertIdentifier,
		FindingID:                 scope.FindingID,
		FindingIDs:                []string{},
		FindingsSummary:           "Analysis incomplete (scope resolution failed)",
		ErrorCount:                1,
		Status:                    "error",
		ErrorSummary:              summary,
		ErrorDetail:               detail,
	}
	p.mu.Lock()
	p.lastActivity = completedAt
	p.lastDuration = record.Duration
	p.resourcesChecked = 0
	p.errorCount = 1
	p.mu.Unlock()
	p.runHistoryStore.Add(record)
}

func patrolLogResourceIDs(ids []string) []string {
	if len(ids) <= scopedPatrolLogIDLimit {
		return ids
	}

	trimmed := append([]string(nil), ids[:scopedPatrolLogIDLimit]...)
	trimmed = append(trimmed, fmt.Sprintf("... +%d more", len(ids)-scopedPatrolLogIDLimit))
	return trimmed
}

type patrolScopeMatcher struct {
	resourceIDSet map[string]bool
	typeSet       map[string]bool
	hasIDs        bool
	hasTypes      bool
	identityOnly  bool
}

func newPatrolScopeMatcher(scope PatrolScope) patrolScopeMatcher {
	resourceIDSet := make(map[string]bool)
	for _, id := range scope.ResourceIDs {
		trimmed := strings.TrimSpace(id)
		if trimmed == "" {
			continue
		}
		resourceIDSet[trimmed] = true
	}

	typeSet := make(map[string]bool)
	addScopeType := func(t string) {
		trimmed := strings.TrimSpace(strings.ToLower(t))
		if trimmed == "" {
			return
		}
		switch trimmed {
		case "docker-host", "app-container":
			typeSet["docker-host"] = true
			typeSet["app-container"] = true
		case "k8s-cluster":
			typeSet["k8s-cluster"] = true
		case "system-container", "vm", "node", "storage", "agent", "pbs", "pmg", "physical_disk":
			typeSet[trimmed] = true
		default:
			typeSet[trimmed] = true
		}
	}
	for _, t := range scope.ResourceTypes {
		addScopeType(t)
	}

	return patrolScopeMatcher{
		resourceIDSet: resourceIDSet,
		typeSet:       typeSet,
		hasIDs:        len(resourceIDSet) > 0,
		hasTypes:      len(typeSet) > 0,
		identityOnly:  scope.resolvedIdentityOnly,
	}
}

func (m patrolScopeMatcher) matchesType(candidates ...string) bool {
	if !m.hasTypes {
		return true
	}
	for _, candidate := range candidates {
		if candidate == "" {
			continue
		}
		if m.typeSet[strings.ToLower(candidate)] {
			return true
		}
	}
	return false
}

func (m patrolScopeMatcher) matchesResource(ids []string, aliases ...string) bool {
	if !m.hasIDs {
		return true
	}
	for _, candidate := range ids {
		if candidate == "" {
			continue
		}
		if m.resourceIDSet[candidate] {
			return true
		}
	}
	if m.identityOnly {
		return false
	}
	for _, candidate := range aliases {
		if candidate != "" && m.resourceIDSet[candidate] {
			return true
		}
	}
	return false
}

type patrolScopedFilterState struct {
	filtered            patrolRuntimeState
	includedResourceIDs map[string]bool
	includedGuestVMIDs  map[int]bool
}

func newPatrolScopedFilterState(snap patrolRuntimeState) patrolScopedFilterState {
	return patrolScopedFilterState{
		filtered: patrolRuntimeState{
			readState:               snap.readState,
			unifiedResourceProvider: snap.unifiedResourceProvider,
			thresholdProvider:       snap.thresholdProvider,
		},
		includedResourceIDs: make(map[string]bool),
		includedGuestVMIDs:  make(map[int]bool),
	}
}

func (s *patrolScopedFilterState) includeResourceID(ids ...string) {
	for _, id := range ids {
		if strings.TrimSpace(id) == "" {
			continue
		}
		s.includedResourceIDs[id] = true
	}
}

func (s *patrolScopedFilterState) includeGuestVMID(vmid int) {
	if vmid > 0 {
		s.includedGuestVMIDs[vmid] = true
	}
}

func patrolDockerScopeName(d models.DockerHost) string {
	hostName := d.CustomDisplayName
	if hostName == "" {
		hostName = d.DisplayName
	}
	if hostName == "" {
		hostName = d.Hostname
	}
	return hostName
}

func scopePatrolDockerHost(d models.DockerHost, matcher patrolScopeMatcher) (models.DockerHost, []string, bool) {
	if !matcher.matchesType("docker-host", "app-container") {
		return models.DockerHost{}, nil, false
	}

	hostMatches := matcher.matchesResource([]string{d.ID}, patrolDockerScopeName(d), d.Hostname, d.DisplayName, d.CustomDisplayName)
	if !matcher.hasIDs {
		included := make([]string, 0, len(d.Containers)+1)
		if matcher.typeSet["docker-host"] || !matcher.hasTypes {
			included = append(included, d.ID)
		}
		if matcher.typeSet["app-container"] || !matcher.hasTypes {
			for _, c := range d.Containers {
				included = append(included, c.ID)
			}
		}
		return d, included, true
	}

	matchedContainers := make([]models.DockerContainer, 0)
	for _, c := range d.Containers {
		if matcher.matchesResource([]string{c.ID}, c.Name) {
			matchedContainers = append(matchedContainers, c)
		}
	}

	if hostMatches {
		included := make([]string, 0, len(d.Containers)+1)
		if matcher.typeSet["docker-host"] || !matcher.hasTypes {
			included = append(included, d.ID)
		}
		if matcher.typeSet["app-container"] || !matcher.hasTypes {
			for _, c := range d.Containers {
				included = append(included, c.ID)
			}
		}
		return d, included, true
	}
	if len(matchedContainers) > 0 {
		hostCopy := d
		hostCopy.Containers = matchedContainers
		// A Docker host is the transport parent for its containers, so the
		// filtered snapshot must retain the host record. It must not, however,
		// retain the host's complete child inventory. Re-ingesting that inventory
		// into the scoped unified-resource provider would turn a request for one
		// container into every image, volume, network, and Swarm object on the
		// host. The matched container already carries its own image, mounts, and
		// network attachments as bounded dependency evidence.
		hostCopy.Images = nil
		hostCopy.Volumes = nil
		hostCopy.Networks = nil
		hostCopy.Services = nil
		hostCopy.Tasks = nil
		hostCopy.Nodes = nil
		hostCopy.Secrets = nil
		hostCopy.Configs = nil
		hostCopy.StorageUsage = nil
		hostCopy.Swarm = nil
		included := make([]string, 0, len(matchedContainers))
		for _, c := range matchedContainers {
			included = append(included, c.ID)
		}
		return hostCopy, included, true
	}

	return models.DockerHost{}, nil, false
}

func patrolUpdateSafetyDockerHosts(s patrolRuntimeState) []models.DockerHost {
	rs := s.readState
	if rs == nil {
		derived := s.withDerivedProviders()
		rs = derived.readState
	}
	if rs == nil {
		return nil
	}

	hostViews := rs.DockerHosts()
	hosts := make([]models.DockerHost, 0, len(hostViews))
	for _, hostView := range hostViews {
		if hostView == nil {
			continue
		}

		hostID := strings.TrimSpace(hostView.HostSourceID())
		if hostID == "" {
			hostID = strings.TrimSpace(hostView.ID())
		}
		name := strings.TrimSpace(hostView.Name())
		displayName := strings.TrimSpace(hostView.DisplayName())
		if displayName == "" {
			displayName = name
		}
		hostname := strings.TrimSpace(hostView.Hostname())
		if hostname == "" {
			hostname = name
		}

		hosts = append(hosts, models.DockerHost{
			ID:                hostID,
			AgentID:           strings.TrimSpace(hostView.AgentID()),
			Hostname:          hostname,
			DisplayName:       displayName,
			CustomDisplayName: strings.TrimSpace(hostView.CustomDisplayName()),
			Status:            string(hostView.Status()),
			Containers:        hostView.Containers(),
		})
	}
	return hosts
}

func collectPatrolScopedDockerHosts(hosts []models.DockerHost, matcher patrolScopeMatcher) ([]models.DockerHost, []string) {
	filtered := make([]models.DockerHost, 0, len(hosts))
	ids := make([]string, 0, len(hosts))
	for _, host := range hosts {
		scopedHost, includeIDs, ok := scopePatrolDockerHost(host, matcher)
		if !ok {
			continue
		}
		filtered = append(filtered, scopedHost)
		ids = append(ids, includeIDs...)
	}
	return filtered, ids
}

func scopePatrolPBSInstance(pbs models.PBSInstance, matcher patrolScopeMatcher) bool {
	if !matcher.matchesType("pbs") {
		return false
	}

	pbsName := pbs.Name
	if pbsName == "" {
		pbsName = pbs.Host
	}
	pbsMatches := matcher.matchesResource([]string{pbs.ID}, pbs.Name, pbsName, pbs.Host)
	if !matcher.hasIDs {
		return true
	}
	if !pbsMatches {
		for _, ds := range pbs.Datastores {
			if matcher.matchesResource([]string{pbs.ID + ":" + ds.Name}, ds.Name) {
				pbsMatches = true
				break
			}
		}
	}
	if !pbsMatches {
		for _, job := range pbs.BackupJobs {
			if matcher.matchesResource([]string{pbs.ID + ":job:" + job.ID}, job.ID) {
				pbsMatches = true
				break
			}
		}
	}
	if !pbsMatches {
		for _, job := range pbs.VerifyJobs {
			if matcher.matchesResource([]string{pbs.ID + ":verify:" + job.ID}, job.ID) {
				pbsMatches = true
				break
			}
		}
	}
	return pbsMatches
}

func scopePatrolNode(n models.Node, matcher patrolScopeMatcher) bool {
	return matcher.matchesType("node") && matcher.matchesResource([]string{n.ID}, n.Name)
}

func scopePatrolVM(vm models.VM, matcher patrolScopeMatcher) bool {
	return matcher.matchesType("vm") && matcher.matchesResource([]string{vm.ID}, vm.Name)
}

func scopePatrolContainer(ct models.Container, matcher patrolScopeMatcher) bool {
	return matcher.matchesType("system-container") && matcher.matchesResource([]string{ct.ID}, ct.Name)
}

func scopePatrolStorage(storage models.Storage, matcher patrolScopeMatcher) bool {
	return matcher.matchesType("storage") && matcher.matchesResource([]string{storage.ID}, storage.Name)
}

func scopePatrolPhysicalDisk(disk models.PhysicalDisk, matcher patrolScopeMatcher) bool {
	return matcher.matchesType("physical_disk") && matcher.matchesResource([]string{disk.ID, disk.DevPath}, disk.Model)
}

func scopePatrolPMGInstance(pmg models.PMGInstance, matcher patrolScopeMatcher) bool {
	return matcher.matchesType("pmg") && matcher.matchesResource([]string{pmg.ID}, pmg.Name, pmg.Host)
}

func scopePatrolHost(h models.Host, matcher patrolScopeMatcher) bool {
	return matcher.matchesType("agent") && matcher.matchesResource([]string{h.ID}, h.DisplayName, h.Hostname)
}

func scopePatrolKubernetesCluster(k models.KubernetesCluster, matcher patrolScopeMatcher) bool {
	return matcher.matchesType("k8s-cluster") && matcher.matchesResource([]string{k.ID}, patrolKubernetesScopeName(k))
}

func collectPatrolScopedNodes(nodes []models.Node, matcher patrolScopeMatcher) ([]models.Node, []string) {
	filtered := make([]models.Node, 0, len(nodes))
	ids := make([]string, 0, len(nodes))
	for _, n := range nodes {
		if !scopePatrolNode(n, matcher) {
			continue
		}
		filtered = append(filtered, n)
		ids = append(ids, n.ID)
	}
	return filtered, ids
}

func collectPatrolScopedVMs(vms []models.VM, matcher patrolScopeMatcher) ([]models.VM, []string, []int) {
	filtered := make([]models.VM, 0, len(vms))
	ids := make([]string, 0, len(vms))
	vmids := make([]int, 0, len(vms))
	for _, vm := range vms {
		if !scopePatrolVM(vm, matcher) {
			continue
		}
		filtered = append(filtered, vm)
		ids = append(ids, vm.ID)
		vmids = append(vmids, vm.VMID)
	}
	return filtered, ids, vmids
}

func collectPatrolScopedContainers(containers []models.Container, matcher patrolScopeMatcher) ([]models.Container, []string, []int) {
	filtered := make([]models.Container, 0, len(containers))
	ids := make([]string, 0, len(containers))
	vmids := make([]int, 0, len(containers))
	for _, ct := range containers {
		if !scopePatrolContainer(ct, matcher) {
			continue
		}
		filtered = append(filtered, ct)
		ids = append(ids, ct.ID)
		vmids = append(vmids, ct.VMID)
	}
	return filtered, ids, vmids
}

func collectPatrolScopedStorage(storage []models.Storage, matcher patrolScopeMatcher) ([]models.Storage, []string) {
	filtered := make([]models.Storage, 0, len(storage))
	ids := make([]string, 0, len(storage))
	for _, s := range storage {
		if !scopePatrolStorage(s, matcher) {
			continue
		}
		filtered = append(filtered, s)
		ids = append(ids, s.ID)
	}
	return filtered, ids
}

func collectPatrolScopedPhysicalDisks(disks []models.PhysicalDisk, matcher patrolScopeMatcher) ([]models.PhysicalDisk, []string) {
	filtered := make([]models.PhysicalDisk, 0, len(disks))
	ids := make([]string, 0, len(disks)*2)
	for _, disk := range disks {
		if !scopePatrolPhysicalDisk(disk, matcher) {
			continue
		}
		filtered = append(filtered, disk)
		ids = append(ids, disk.ID, disk.DevPath)
	}
	return filtered, ids
}

func collectPatrolScopedPBSInstances(instances []models.PBSInstance, matcher patrolScopeMatcher) ([]models.PBSInstance, []string) {
	filtered := make([]models.PBSInstance, 0, len(instances))
	ids := make([]string, 0, len(instances))
	for _, pbs := range instances {
		if !scopePatrolPBSInstance(pbs, matcher) {
			continue
		}
		filtered = append(filtered, pbs)
		ids = append(ids, pbs.ID)
	}
	return filtered, ids
}

func collectPatrolScopedPMGInstances(instances []models.PMGInstance, matcher patrolScopeMatcher) ([]models.PMGInstance, []string) {
	filtered := make([]models.PMGInstance, 0, len(instances))
	ids := make([]string, 0, len(instances))
	for _, pmg := range instances {
		if !scopePatrolPMGInstance(pmg, matcher) {
			continue
		}
		filtered = append(filtered, pmg)
		ids = append(ids, pmg.ID)
	}
	return filtered, ids
}

func collectPatrolScopedHosts(hosts []models.Host, matcher patrolScopeMatcher) ([]models.Host, []string) {
	filtered := make([]models.Host, 0, len(hosts))
	ids := make([]string, 0, len(hosts))
	for _, h := range hosts {
		if !scopePatrolHost(h, matcher) {
			continue
		}
		filtered = append(filtered, h)
		ids = append(ids, h.ID)
	}
	return filtered, ids
}

func collectPatrolScopedKubernetesClusters(clusters []models.KubernetesCluster, matcher patrolScopeMatcher) ([]models.KubernetesCluster, []string) {
	filtered := make([]models.KubernetesCluster, 0, len(clusters))
	ids := make([]string, 0, len(clusters))
	for _, k := range clusters {
		if !scopePatrolKubernetesCluster(k, matcher) {
			continue
		}
		filtered = append(filtered, k)
		ids = append(ids, k.ID)
	}
	return filtered, ids
}

func patrolUnifiedResourceIdentity(resource unifiedresources.Resource) ([]string, []string) {
	ids := []string{resource.ID}
	aliases := []string{resource.Name}
	if resource.Canonical != nil {
		ids = append(ids, resource.Canonical.PrimaryID)
		ids = append(ids, resource.Canonical.SupersededIDs...)
		aliases = append(aliases, resource.Canonical.DisplayName, resource.Canonical.Hostname)
		aliases = append(aliases, resource.Canonical.Aliases...)
	}
	ids = append(ids, resource.SupersededCanonicalIDs...)
	return ids, aliases
}

func collectPatrolScopedUnifiedResources(provider UnifiedResourceProvider, matcher patrolScopeMatcher) ([]unifiedresources.Resource, []string) {
	if provider == nil {
		return nil, nil
	}
	filtered := make([]unifiedresources.Resource, 0)
	ids := make([]string, 0)
	for _, resource := range provider.GetAll() {
		resourceIDs, aliases := patrolUnifiedResourceIdentity(resource)
		if !matcher.matchesType(string(resource.Type)) || !matcher.matchesResource(resourceIDs, aliases...) {
			continue
		}
		filtered = append(filtered, resource)
		ids = append(ids, resourceIDs...)
	}
	return filtered, ids
}

func patrolKubernetesScopeName(k models.KubernetesCluster) string {
	clusterName := k.CustomDisplayName
	if clusterName == "" {
		clusterName = k.DisplayName
	}
	if clusterName == "" {
		clusterName = k.Name
	}
	return clusterName
}

func collectPatrolScopedActiveAlerts(state patrolRuntimeState, alerts []models.Alert, includedResourceIDs map[string]bool) []models.Alert {
	filtered := make([]models.Alert, 0, len(alerts))
	for _, alert := range alerts {
		if patrolAlertBelongsToScope(state, alert.ResourceID, includedResourceIDs) {
			filtered = append(filtered, alert)
		}
	}
	return filtered
}

func collectPatrolScopedResolvedAlerts(state patrolRuntimeState, alerts []models.ResolvedAlert, includedResourceIDs map[string]bool) []models.ResolvedAlert {
	filtered := make([]models.ResolvedAlert, 0, len(alerts))
	for _, resolved := range alerts {
		if patrolAlertBelongsToScope(state, resolved.ResourceID, includedResourceIDs) {
			filtered = append(filtered, resolved)
		}
	}
	return filtered
}

func collectPatrolScopedConnectionHealth(connectionHealth map[string]bool, includedResourceIDs map[string]bool) map[string]bool {
	filtered := make(map[string]bool, len(connectionHealth))
	for resourceID, healthy := range connectionHealth {
		if includedResourceIDs[resourceID] {
			filtered[resourceID] = healthy
		}
	}
	return filtered
}

func collectPatrolScopedBackupTasks(tasks []models.BackupTask, includedGuestVMIDs map[int]bool) []models.BackupTask {
	filtered := make([]models.BackupTask, 0, len(tasks))
	for _, backupTask := range tasks {
		if includedGuestVMIDs[backupTask.VMID] {
			filtered = append(filtered, backupTask)
		}
	}
	return filtered
}

func collectPatrolScopedStorageBackups(backups []models.StorageBackup, includedGuestVMIDs map[int]bool) []models.StorageBackup {
	filtered := make([]models.StorageBackup, 0, len(backups))
	for _, storageBackup := range backups {
		if includedGuestVMIDs[storageBackup.VMID] {
			filtered = append(filtered, storageBackup)
		}
	}
	return filtered
}

func collectPatrolScopedGuestSnapshots(snapshots []models.GuestSnapshot, includedGuestVMIDs map[int]bool) []models.GuestSnapshot {
	filtered := make([]models.GuestSnapshot, 0, len(snapshots))
	for _, guestSnapshot := range snapshots {
		if includedGuestVMIDs[guestSnapshot.VMID] {
			filtered = append(filtered, guestSnapshot)
		}
	}
	return filtered
}

func collectPatrolScopedPBSBackups(backups []models.PBSBackup, includedGuestVMIDs map[int]bool) []models.PBSBackup {
	filtered := make([]models.PBSBackup, 0, len(backups))
	for _, backup := range backups {
		vmid, err := strconv.Atoi(backup.VMID)
		if err == nil && includedGuestVMIDs[vmid] {
			filtered = append(filtered, backup)
		}
	}
	return filtered
}

func copyScopedPatrolMetadata(dst *patrolRuntimeState, snap patrolRuntimeState, includedResourceIDs map[string]bool, includedGuestVMIDs map[int]bool) {
	if len(snap.ActiveAlerts) > 0 {
		dst.ActiveAlerts = collectPatrolScopedActiveAlerts(snap, snap.ActiveAlerts, includedResourceIDs)
	}
	if len(snap.RecentlyResolved) > 0 {
		dst.RecentlyResolved = collectPatrolScopedResolvedAlerts(snap, snap.RecentlyResolved, includedResourceIDs)
	}
	if len(snap.ConnectionHealth) > 0 {
		dst.ConnectionHealth = collectPatrolScopedConnectionHealth(snap.ConnectionHealth, includedResourceIDs)
	}
	if len(includedGuestVMIDs) == 0 {
		return
	}

	dst.PVEBackups.BackupTasks = collectPatrolScopedBackupTasks(snap.PVEBackups.BackupTasks, includedGuestVMIDs)
	dst.PVEBackups.StorageBackups = collectPatrolScopedStorageBackups(snap.PVEBackups.StorageBackups, includedGuestVMIDs)
	dst.PVEBackups.GuestSnapshots = collectPatrolScopedGuestSnapshots(snap.PVEBackups.GuestSnapshots, includedGuestVMIDs)
	dst.PBSBackups = collectPatrolScopedPBSBackups(snap.PBSBackups, includedGuestVMIDs)
}

func filterPatrolStateByScopeState(snap patrolRuntimeState, scope PatrolScope) patrolRuntimeState {
	if len(snap.ActiveAlerts)+len(snap.RecentlyResolved) > 0 {
		snap.alertResourceResolver = patrolAlertReferenceResolver(snap)
	}
	matcher := newPatrolScopeMatcher(scope)
	filterState := newPatrolScopedFilterState(snap)

	filteredNodes, nodeIDs := collectPatrolScopedNodes(snap.Nodes, matcher)
	filterState.filtered.Nodes = filteredNodes
	filterState.includeResourceID(nodeIDs...)

	filteredVMs, vmIDs, guestVMIDs := collectPatrolScopedVMs(snap.VMs, matcher)
	filterState.filtered.VMs = filteredVMs
	filterState.includeResourceID(vmIDs...)
	for _, vmid := range guestVMIDs {
		filterState.includeGuestVMID(vmid)
	}

	filteredContainers, containerIDs, containerVMIDs := collectPatrolScopedContainers(snap.Containers, matcher)
	filterState.filtered.Containers = filteredContainers
	filterState.includeResourceID(containerIDs...)
	for _, vmid := range containerVMIDs {
		filterState.includeGuestVMID(vmid)
	}

	filteredDockerHosts, dockerIDs := collectPatrolScopedDockerHosts(snap.DockerHosts, matcher)
	filterState.filtered.DockerHosts = filteredDockerHosts
	filterState.includeResourceID(dockerIDs...)

	filteredStorage, storageIDs := collectPatrolScopedStorage(snap.Storage, matcher)
	filterState.filtered.Storage = filteredStorage
	filterState.includeResourceID(storageIDs...)

	filteredDisks, diskIDs := collectPatrolScopedPhysicalDisks(snap.PhysicalDisks, matcher)
	filterState.filtered.PhysicalDisks = filteredDisks
	filterState.includeResourceID(diskIDs...)

	filteredPBS, pbsIDs := collectPatrolScopedPBSInstances(snap.PBSInstances, matcher)
	filterState.filtered.PBSInstances = filteredPBS
	filterState.includeResourceID(pbsIDs...)

	filteredPMG, pmgIDs := collectPatrolScopedPMGInstances(snap.PMGInstances, matcher)
	filterState.filtered.PMGInstances = filteredPMG
	filterState.includeResourceID(pmgIDs...)

	filteredHosts, hostIDs := collectPatrolScopedHosts(snap.Hosts, matcher)
	filterState.filtered.Hosts = filteredHosts
	filterState.includeResourceID(hostIDs...)

	filteredK8sClusters, k8sClusterIDs := collectPatrolScopedKubernetesClusters(snap.KubernetesClusters, matcher)
	filterState.filtered.KubernetesClusters = filteredK8sClusters
	filterState.includeResourceID(k8sClusterIDs...)

	// Preserve exact canonical resources that have no StateSnapshot projection
	// (notably availability network endpoints). Rebuilding the provider only
	// from snapshot fields made valid objective scopes disappear before the
	// model could propose a local observer. The scoped provider is still built
	// from the exact matcher result, so unrelated resources cannot enter model
	// context.
	filteredUnified, unifiedIDs := collectPatrolScopedUnifiedResources(snap.unifiedResourceProvider, matcher)
	filterState.includeResourceID(unifiedIDs...)
	if len(filteredUnified) > 0 {
		filterState.filtered.alertResourceResolver = patrolAlertReferenceResolver(snap)
	}

	copyScopedPatrolMetadata(&filterState.filtered, snap, filterState.includedResourceIDs, filterState.includedGuestVMIDs)

	return filterState.filtered.withDerivedProvidersAndResources(filteredUnified)
}

func (p *PatrolService) filterStateByScopeState(snap patrolRuntimeState, scope PatrolScope) patrolRuntimeState {
	return filterPatrolStateByScopeState(snap, scope)
}

// GetStatus returns the current patrol status
func (p *PatrolService) GetStatus() PatrolStatus {
	p.mu.RLock()
	defer p.mu.RUnlock()

	interval := p.config.GetInterval()
	intervalMs := int64(interval / time.Millisecond)

	// "Running" means an analysis is currently in progress, not just the service loop
	analysisInProgress := p.runInProgress

	status := PatrolStatus{
		RuntimeState:     PatrolRuntimeStateActive,
		Running:          analysisInProgress,
		CurrentRunID:     p.currentRunID,
		Enabled:          p.config.Enabled,
		LastDuration:     p.lastDuration,
		ResourcesChecked: p.resourcesChecked,
		FindingsCount:    len(p.findings.GetActive(FindingSeverityInfo)),
		ErrorCount:       p.errorCount,
		IntervalMs:       intervalMs,
		BlockedReason:    p.lastBlockedReason,
		BlockedCause:     p.lastBlockedCause,
	}
	if analysisInProgress && !p.runStartedAt.IsZero() {
		startedAt := p.runStartedAt
		status.CurrentRunStartedAt = &startedAt
	}
	if p.triggerManager != nil {
		triggerStatus := p.triggerManager.GetStatus()
		status.TriggerStatus = &triggerStatus
	}

	switch {
	case analysisInProgress:
		status.RuntimeState = PatrolRuntimeStateRunning
	case !p.config.Enabled:
		status.RuntimeState = PatrolRuntimeStateDisabled
	case strings.TrimSpace(status.BlockedReason) != "":
		status.RuntimeState = PatrolRuntimeStateBlocked
	default:
		status.RuntimeState = PatrolRuntimeStateActive
	}

	if !p.lastFullPatrol.IsZero() {
		status.LastPatrolAt = &p.lastFullPatrol
	}
	if !p.lastActivity.IsZero() {
		status.LastActivityAt = &p.lastActivity
	}
	if strings.TrimSpace(status.BlockedReason) != "" && !p.lastBlockedAt.IsZero() {
		status.BlockedAt = &p.lastBlockedAt
	}

	// Use the tracked next scheduled time (accounts for ticker resets on interval changes)
	if p.config.Enabled && interval > 0 && !p.nextScheduledAt.IsZero() {
		next := p.nextScheduledAt
		status.NextPatrolAt = &next
	}

	summary := p.findings.GetSummary()
	status.Healthy = summary.IsHealthy()
	if status.RuntimeState == PatrolRuntimeStateBlocked {
		status.Healthy = false
	}

	return status
}

// SubscribeToStream returns a channel that will receive streaming patrol events
func (p *PatrolService) SubscribeToStream() chan PatrolStreamEvent {
	return p.SubscribeToStreamFrom(0)
}

// SubscribeToStreamFrom subscribes a client to patrol streaming events and optionally replays
// events with Seq > lastSeq (best-effort). This allows SSE clients to resume after disconnects
// using the Last-Event-ID header.
func (p *PatrolService) SubscribeToStreamFrom(lastSeq int64) chan PatrolStreamEvent {
	ch := make(chan PatrolStreamEvent, 100) // Buffered to prevent blocking
	sub := &streamSubscriber{ch: ch}
	replayedCount := 0
	snapshotReasons := make([]string, 0, 2)
	snapshotReasonSeen := make(map[string]struct{}, 2)

	p.streamMu.Lock()
	p.streamSubscribers[ch] = sub

	trySendSnapshot := func(reason string) bool {
		if _, seen := snapshotReasonSeen[reason]; seen {
			return true
		}
		snap := p.makeSnapshotLocked(reason)
		select {
		case ch <- snap:
			snapshotReasonSeen[reason] = struct{}{}
			snapshotReasons = append(snapshotReasons, reason)
			return true
		default:
			return false
		}
	}

	bufferStart, bufferEnd := p.streamBufferWindowLocked()
	// If the client is behind the buffered window, proactively emit a snapshot that
	// advertises truncation. (We may still replay what we have.)
	if lastSeq > 0 && bufferStart > 0 && lastSeq < bufferStart && p.streamPhase != "idle" {
		trySendSnapshot("buffer_rotated")
	}

	// Best-effort replay / snapshot:
	// - If client provides lastSeq, replay newer buffered events (Seq > lastSeq).
	// - If lastSeq is stale/ahead (e.g. from a different run), send a snapshot so UI can resync.
	// - If no lastSeq, send a snapshot (late-joiner).
	if lastSeq > 0 && len(p.streamEvents) > 0 {
		events := p.streamEventsSinceLocked(lastSeq)
	replayLoop:
		for _, ev := range events {
			select {
			case ch <- ev:
				replayedCount++
			default:
				// If subscriber can't catch up, stop replaying and let it receive live events.
				break replayLoop
			}
		}
	}
	if replayedCount == 0 && len(snapshotReasons) == 0 && lastSeq > 0 && p.streamPhase != "idle" {
		// lastSeq is likely stale (ahead of this run) or we're missing buffered events.
		// Provide a snapshot to allow the UI to resync.
		reason := "stale_last_event_id"
		if bufferEnd > 0 && lastSeq > bufferEnd {
			reason = "stale_last_event_id"
		} else if bufferStart > 0 && lastSeq < bufferStart {
			reason = "buffer_rotated"
		}
		trySendSnapshot(reason)
	}
	if lastSeq == 0 && p.streamPhase != "idle" {
		trySendSnapshot("late_joiner")
	}
	p.streamMu.Unlock()

	metrics := GetPatrolMetrics()
	if replayedCount > 0 {
		metrics.RecordStreamReplay(replayedCount)
		log.Debug().Int64("last_seq", lastSeq).Int("replayed_events", replayedCount).Msg("patrol stream replayed buffered events")
	}
	for _, reason := range snapshotReasons {
		metrics.RecordStreamSnapshot(reason)
		log.Debug().Int64("last_seq", lastSeq).Str("resync_reason", reason).Msg("patrol stream sent synthetic snapshot")
	}
	if lastSeq > 0 && replayedCount == 0 && len(snapshotReasons) == 0 {
		metrics.RecordStreamMiss()
		log.Debug().Int64("last_seq", lastSeq).Msg("patrol stream resume had no replay or snapshot")
	}

	return ch
}

// UnsubscribeFromStream removes a subscriber
func (p *PatrolService) UnsubscribeFromStream(ch chan PatrolStreamEvent) {
	p.streamMu.Lock()
	sub, exists := p.streamSubscribers[ch]
	delete(p.streamSubscribers, ch)
	p.streamMu.Unlock()

	// Use atomic CAS to ensure exactly one goroutine closes the channel,
	// even if broadcast and unsubscribe race.
	if exists && sub.closed.CompareAndSwap(false, true) {
		close(ch)
	}
}

// broadcast sends an event to all subscribers
// Subscribers with full channels are automatically removed to prevent memory leaks
func (p *PatrolService) broadcast(event PatrolStreamEvent) {
	p.streamMu.Lock()
	defer p.streamMu.Unlock()

	// Track a couple pieces of best-effort state for snapshots/resync.
	switch event.Type {
	case "tool_start":
		if event.ToolName != "" {
			p.streamCurrentTool = event.ToolName
		}
	case "tool_end":
		p.streamCurrentTool = ""
	}

	// Bound payload sizes so streaming and replay buffers can't balloon due to a single tool
	// output or oversized content chunk.
	event = truncateStreamEvent(event)

	// Decorate once so every subscriber sees identical meta.
	event = p.decorateStreamEventLocked(event)
	p.appendStreamEventLocked(event)

	var staleChannels []chan PatrolStreamEvent
	dropReasons := make(map[chan PatrolStreamEvent]string)
	for ch, sub := range p.streamSubscribers {
		if sub == nil || sub.closed.Load() {
			staleChannels = append(staleChannels, ch)
			dropReasons[ch] = "closed"
			continue
		}
		select {
		case ch <- event:
			// Successfully sent
			sub.fullCount = 0
		default:
			// Channel full. Tolerate bursts, but disconnect subscribers that are
			// consistently unable to receive events (likely dead/slow clients).
			sub.fullCount++
			if sub.fullCount >= 25 {
				staleChannels = append(staleChannels, ch)
				dropReasons[ch] = "backpressure"
			}
		}
	}

	// Clean up stale subscribers using atomic CAS for safe close
	for _, ch := range staleChannels {
		sub := p.streamSubscribers[ch]
		delete(p.streamSubscribers, ch)
		reason := dropReasons[ch]
		GetPatrolMetrics().RecordStreamSubscriberDrop(reason)
		log.Debug().Str("reason", reason).Msg("patrol stream subscriber dropped")
		if sub != nil && sub.closed.CompareAndSwap(false, true) {
			close(ch)
		}
	}
}

// resetStreamForRun resets stream state for a new run so late-joiners don't see stale output.
// This should only be called for runs that will actually stream events (NoStream=false).
func (p *PatrolService) resetStreamForRun(runID string) {
	p.streamMu.Lock()
	p.streamRunID = runID
	p.streamSeq = 0
	p.streamPhase = "idle"
	p.streamCurrentTool = ""
	p.currentOutput.Reset()
	p.streamEvents = nil
	p.streamMu.Unlock()
}

func (p *PatrolService) decorateStreamEventLocked(event PatrolStreamEvent) PatrolStreamEvent {
	if event.RunID == "" {
		event.RunID = p.streamRunID
	}
	if event.Seq == 0 {
		p.streamSeq++
		event.Seq = p.streamSeq
	}
	if event.TsMs == 0 {
		event.TsMs = time.Now().UnixMilli()
	}
	return event
}

const patrolStreamReplayBufferSize = 200
const patrolStreamMaxEventFieldBytes = 8 * 1024

func truncateStreamEvent(event PatrolStreamEvent) PatrolStreamEvent {
	event.Content = truncateStreamField(event.Content, patrolStreamMaxEventFieldBytes)
	event.ToolInput = truncateStreamField(event.ToolInput, patrolStreamMaxEventFieldBytes)
	event.ToolRawInput = truncateStreamField(event.ToolRawInput, patrolStreamMaxEventFieldBytes)
	event.ToolOutput = truncateStreamField(event.ToolOutput, patrolStreamMaxEventFieldBytes)
	return event
}

func truncateStreamField(s string, max int) string {
	if max <= 0 || len(s) <= max {
		return s
	}
	const suffix = "...[truncated]"
	if max <= len(suffix) {
		return s[:max]
	}
	return s[:max-len(suffix)] + suffix
}

func (p *PatrolService) appendStreamEventLocked(event PatrolStreamEvent) {
	// Keep a bounded buffer for Last-Event-ID replay (best-effort).
	p.streamEvents = append(p.streamEvents, event)
	if len(p.streamEvents) > patrolStreamReplayBufferSize {
		p.streamEvents = p.streamEvents[len(p.streamEvents)-patrolStreamReplayBufferSize:]
	}
}

func (p *PatrolService) streamEventsSinceLocked(lastSeq int64) []PatrolStreamEvent {
	// Seq is monotonic within a run; we reset buffer on new run.
	for i := len(p.streamEvents) - 1; i >= 0; i-- {
		if p.streamEvents[i].Seq <= lastSeq {
			// Return events after i
			out := make([]PatrolStreamEvent, len(p.streamEvents)-(i+1))
			copy(out, p.streamEvents[i+1:])
			return out
		}
	}
	// All buffered events are newer
	out := make([]PatrolStreamEvent, len(p.streamEvents))
	copy(out, p.streamEvents)
	return out
}

func (p *PatrolService) streamBufferWindowLocked() (start, end int64) {
	if len(p.streamEvents) == 0 {
		return 0, 0
	}
	return p.streamEvents[0].Seq, p.streamEvents[len(p.streamEvents)-1].Seq
}

func (p *PatrolService) makeSnapshotLocked(reason string) PatrolStreamEvent {
	start, end := p.streamBufferWindowLocked()
	phase := p.streamPhase
	if phase == "idle" {
		phase = ""
	}
	tr := p.currentOutput.Truncated()
	var trPtr *bool
	if tr {
		trPtr = &tr
	}
	// Snapshot is synthetic and should not advance seq; use the most recent real event seq
	// so clients can resume from a meaningful Last-Event-ID.
	seq := end
	return PatrolStreamEvent{
		Type:             "snapshot",
		RunID:            p.streamRunID,
		Seq:              seq,
		TsMs:             time.Now().UnixMilli(),
		ResyncReason:     reason,
		BufferStart:      start,
		BufferEnd:        end,
		ContentTruncated: trPtr,
		Phase:            phase,
		Content:          p.currentOutput.String(),
		ToolName:         p.streamCurrentTool,
	}
}

// appendStreamContent adds content to the current output and broadcasts it
func (p *PatrolService) appendStreamContent(content string) {
	p.streamMu.Lock()
	p.currentOutput.WriteString(content)
	p.streamMu.Unlock()

	p.broadcast(PatrolStreamEvent{
		Type:    "content",
		Content: content,
	})
}

// setStreamPhase updates the current phase and broadcasts it to all subscribers.
// The frontend only updates its phase display when it receives a 'phase' event,
// so we must broadcast phase changes to keep the UI in sync.
func (p *PatrolService) setStreamPhase(phase string) {
	p.streamMu.Lock()
	oldPhase := p.streamPhase
	p.streamPhase = phase
	p.streamMu.Unlock()

	// Broadcast phase change (except for idle which just clears state)
	// This ensures late joiners and continuous watchers see the current phase
	if phase != "idle" && phase != oldPhase {
		p.broadcast(PatrolStreamEvent{
			Type:  "phase",
			Phase: phase,
		})
	}
}

// GetCurrentStreamOutput returns the current buffered output (for late joiners)
func (p *PatrolService) GetCurrentStreamOutput() (string, string) {
	p.streamMu.RLock()
	defer p.streamMu.RUnlock()
	return p.currentOutput.String(), p.streamPhase
}

// reviewAndResolveAlerts uses AI to review active alerts and resolve those where the issue is fixed.
// This is the core of autonomous alert management - the AI looks at each alert, checks current state,
// and determines if the underlying issue has been resolved.
//
// Two structural cost controls run before any model call:
//   - alerts whose trigger condition demonstrably still holds in the current
//     snapshot (metric still at/above threshold, resource still offline,
//     Docker container still unhealthy, stopped or OOM-killed) are skipped —
//     a correct review could only answer KEEP, and a wrong RESOLVE would be
//     raised again by the alert engine on the resource's next report. The
//     question is not worth a billed call. Resolution stays model-owned:
//     Pulse never locally decides that an issue HAS cleared.
//   - the remaining candidates are reviewed in batched model calls instead
//     of one call per alert.
func (p *PatrolService) reviewAndResolveAlertsState(ctx context.Context, state patrolRuntimeState, llmAllowed bool, executionID string) int {
	p.mu.RLock()
	resolver := p.alertResolver
	aiService := p.aiService
	p.mu.RUnlock()

	if resolver == nil {
		return 0
	}

	activeAlerts := resolver.GetActiveAlerts()
	if len(activeAlerts) == 0 {
		return 0
	}

	aiSvc := aiService
	if !llmAllowed {
		aiSvc = nil
	}
	if aiSvc == nil || !aiSvc.IsEnabled() {
		return 0
	}
	if state.alertResourceResolver == nil {
		// Built once: each review context may consult it.
		state.alertResourceResolver = patrolAlertReferenceResolver(state)
	}

	// Only review alerts that have been active for at least 10 minutes
	// This avoids thrashing on transient alerts
	minAge := 10 * time.Minute
	var alertsToReview []AlertInfo
	stillFiring := 0
	for _, alert := range activeAlerts {
		if time.Since(alert.StartTime) < minAge {
			continue
		}
		if p.alertStillFiringState(alert, state) {
			stillFiring++
			continue
		}
		alertsToReview = append(alertsToReview, alert)
	}

	if len(alertsToReview) == 0 {
		if stillFiring > 0 {
			log.Debug().
				Int("still_firing", stillFiring).
				Msg("AI Patrol: All aged alerts still demonstrably firing, skipping model review")
		}
		return 0
	}

	log.Info().
		Int("total_active", len(activeAlerts)).
		Int("to_review", len(alertsToReview)).
		Int("still_firing_skipped", stillFiring).
		Msg("AI Patrol: Reviewing alerts for auto-resolution")

	resolvedCount := 0
	for _, verdict := range p.reviewAlertsWithModelState(ctx, alertsToReview, state, aiSvc, executionID) {
		if !verdict.resolve {
			continue
		}
		if resolver.ResolveAlert(verdict.alert.ID) {
			resolvedCount++
			log.Info().
				Str("alertID", verdict.alert.ID).
				Str("resource", verdict.alert.ResourceName).
				Str("reason", verdict.reason).
				Dur("age", time.Since(verdict.alert.StartTime)).
				Msg("AI Patrol: Auto-resolved alert - issue no longer detected")
		}
	}

	if resolvedCount > 0 {
		log.Info().
			Int("resolved", resolvedCount).
			Msg("AI Patrol: Completed alert review")
	}

	return resolvedCount
}

// alertStillFiringState reports whether the current runtime snapshot still
// demonstrably shows the alert's trigger condition — a metric alert whose
// current value is still at or above its threshold, an offline alert
// whose resource is still not online, or a Docker container alert whose
// container still reports the state that raised it (for a restart loop,
// a container caught in Docker's restart backoff). For these alerts a
// correct model review can only answer KEEP, so Patrol skips the billed
// call and keeps the alert. The gate never decides that an issue HAS
// cleared: anything uncertain (unknown alert type, missing resource,
// unmapped metric) returns false so the model still owns the resolution
// decision.
func (p *PatrolService) alertStillFiringState(alert AlertInfo, snap patrolRuntimeState) bool {
	switch alert.Type {
	case "cpu", "memory", "usage", "disk":
		if alert.Threshold <= 0 {
			return false
		}
		// getCurrentMetricValueState returns -1 when the resource or metric
		// is unknown, which can never satisfy a positive threshold.
		return p.getCurrentMetricValueState(alert, snap) >= alert.Threshold
	case "offline", "powered-off", "host-offline":
		resource := lookupPatrolAlertResourceState(alert, snap)
		if !resource.found {
			// Resource is gone from the snapshot — the model should judge
			// whether the alert resolves as removed.
			return false
		}
		switch resource.resourceType {
		case "node", "agent", "vm", "system-container":
			return !p.isResourceOnlineState(alert, snap)
		default:
			return false
		}
	case "docker-container-health", "docker-container-state", "docker-container-oom-kill", "docker-container-restart-loop":
		resource := lookupPatrolAlertResourceState(alert, snap)
		if !resource.found || resource.resourceType != "app-container" {
			return false
		}
		return dockerContainerAlertStillFiring(alert.Type, resource)
	default:
		return false
	}
}

// dockerContainerAlertingStates are the runtime states the alert subsystem
// raises docker-container-state for (checkDockerContainerState).
var dockerContainerAlertingStates = map[string]bool{
	"created": true, "restarting": true, "removing": true, "paused": true,
	"exited": true, "dead": true, "unknown": true,
}

// dockerContainerAlertStillFiring mirrors the alert subsystem's Docker
// container health, state and OOM checks (internal/alerts/docker.go)
// against the container's current report. It answers only in the keep
// direction: a container that may have recovered returns false and goes
// to the model.
func dockerContainerAlertStillFiring(alertType string, container patrolAlertResourceState) bool {
	state := strings.ToLower(strings.TrimSpace(container.status))
	switch alertType {
	case "docker-container-health":
		// Health is live evidence only while the container runs; a
		// stopped container's last health result belongs to the state
		// alert, which the alert subsystem raises instead.
		if state != "" && state != "running" {
			return false
		}
		switch strings.ToLower(strings.TrimSpace(container.health)) {
		case "", "none", "healthy", "starting":
			return false
		}
		return true
	case "docker-container-state":
		return dockerContainerAlertingStates[state]
	case "docker-container-oom-kill":
		return (state == "exited" || state == "dead") && container.oomKilled != nil && *container.oomKilled
	case "docker-container-restart-loop":
		// The alert subsystem counts restarts over a window the snapshot
		// does not carry, so this is not its check. A container caught in
		// Docker's restart backoff is still restarting, though; keeping
		// its alert defers to the alert subsystem, which clears it once
		// the window empties.
		return state == "restarting"
	default:
		return false
	}
}

// getCurrentMetricValue gets the current value of the metric that triggered the alert
type patrolAlertResourceState struct {
	resourceType string
	platform     string
	name         string
	status       string
	health       string
	restartCount int
	oomKilled    *bool
	cpu          float64
	memory       float64
	guestMemory  *patrolGuestMemoryReading
	disk         float64
	found        bool
}

func patrolAlertNameMatches(alert AlertInfo, ids ...string) bool {
	for _, id := range ids {
		if strings.TrimSpace(id) == "" {
			continue
		}
		if id == alert.ResourceID || id == alert.ResourceName {
			return true
		}
	}
	return false
}

func patrolAlertLookupType(alert AlertInfo) string {
	resourceType := canonicalizeAICompatibilityResourceType(alert.ResourceType)
	if alert.Type == "usage" && (resourceType == "" || strings.EqualFold(resourceType, "usage")) {
		return "storage"
	}
	return resourceType
}

func lookupPatrolAlertResourceState(alert AlertInfo, snap patrolRuntimeState) patrolAlertResourceState {
	alert.ResourceType = patrolAlertLookupType(alert)
	if alert.ResourceType == "app-container" {
		if resource, ok := patrolLookupAppContainerAlertResourceState(alert, snap); ok {
			return resource
		}
	}
	switch alert.ResourceType {
	case "storage":
		if resource, ok := patrolLookupStorageAlertResourceState(alert, snap); ok {
			return resource
		}
	case "node":
		if resource, ok := patrolLookupNodeAlertResourceState(alert, snap); ok {
			return resource
		}
	case "vm":
		if resource, ok := patrolLookupGuestAlertResourceState(alert, snap, "VM", "vm"); ok {
			return resource
		}
	case "system-container":
		if resource, ok := patrolLookupGuestAlertResourceState(alert, snap, "Container", "system-container"); ok {
			return resource
		}
	case "agent":
		if resource, ok := patrolLookupHostAlertResourceState(alert, snap); ok {
			return resource
		}
	}
	return patrolAlertResourceState{}
}

// patrolAlertSubjectInInventory reports whether current inventory holds the
// alert's subject, or the resource that owns it, by durable identity.
func patrolAlertSubjectInInventory(alert AlertInfo, snap patrolRuntimeState) bool {
	if strings.TrimSpace(alert.ResourceID) == "" {
		return false
	}
	binding, _ := patrolAlertReferenceResolver(snap).ResolveAlertResourceReference(alert.ResourceID)
	return binding.ResourceID != ""
}

func patrolLookupAppContainerAlertResourceState(alert AlertInfo, snap patrolRuntimeState) (patrolAlertResourceState, bool) {
	rows := patrolAppContainerRows(snap, nil)
	// An alert subsystem container ID ("docker:<host>/<container>") names
	// one container on one host, so it alone decides: a display-name match
	// could judge the alert against a same-named container elsewhere.
	exact := strings.HasPrefix(alert.ResourceID, "docker:")
	for _, container := range rows {
		if exact {
			if container.alertResourceID != alert.ResourceID {
				continue
			}
		} else if !patrolAlertNameMatches(alert, container.id, container.name) {
			continue
		}
		return patrolAlertResourceState{
			resourceType: "app-container",
			name:         container.name,
			status:       container.status,
			health:       container.health,
			restartCount: container.restartCount,
			oomKilled:    container.oomKilled,
			cpu:          container.cpu,
			memory:       container.memory,
			found:        true,
		}, true
	}
	return patrolAlertResourceState{}, false
}

func patrolLookupStorageAlertResourceState(alert AlertInfo, snap patrolRuntimeState) (patrolAlertResourceState, bool) {
	for _, storage := range patrolStoragePoolRows(snap, nil) {
		if !patrolAlertNameMatches(alert, storage.id, storage.name) {
			continue
		}
		return patrolAlertResourceState{
			resourceType: "storage",
			name:         storage.name,
			status:       storage.status,
			disk:         storage.usage,
			found:        true,
		}, true
	}
	return patrolAlertResourceState{}, false
}

func patrolLookupNodeAlertResourceState(alert AlertInfo, snap patrolRuntimeState) (patrolAlertResourceState, bool) {
	for _, node := range patrolNodeInventoryRows(snap, nil) {
		if !patrolAlertNameMatches(alert, node.id, node.name) {
			continue
		}
		return patrolAlertResourceState{
			resourceType: "node",
			name:         node.name,
			status:       node.status,
			cpu:          node.cpu,
			memory:       node.mem,
			found:        true,
		}, true
	}
	return patrolAlertResourceState{}, false
}

func patrolLookupGuestAlertResourceState(alert AlertInfo, snap patrolRuntimeState, guestType, resourceType string) (patrolAlertResourceState, bool) {
	for _, guest := range patrolGuestInventoryRows(snap, nil, nil) {
		if guest.gType != guestType || !patrolAlertNameMatches(alert, guest.id, guest.name) {
			continue
		}
		return patrolAlertResourceState{
			resourceType: resourceType,
			name:         guest.name,
			status:       guest.status,
			cpu:          guest.cpu,
			memory:       guest.mem,
			guestMemory:  &guest.memory,
			found:        true,
		}, true
	}
	return patrolAlertResourceState{}, false
}

type patrolHostAlertRow struct {
	id, name, hostname, status, platform string
	cpu, memory                          float64
}

func patrolHostAlertRows(snap patrolRuntimeState) []patrolHostAlertRow {
	if snap.readState != nil {
		hosts := snap.readState.Hosts()
		rows := make([]patrolHostAlertRow, 0, len(hosts))
		for _, host := range hosts {
			rows = append(rows, patrolHostAlertRow{
				id:       host.ID(),
				name:     host.Name(),
				hostname: host.Hostname(),
				status:   string(host.Status()),
				platform: host.Platform(),
				cpu:      host.CPUPercent(),
				memory:   host.MemoryPercent(),
			})
		}
		return rows
	}

	rows := make([]patrolHostAlertRow, 0, len(snap.Hosts))
	for _, host := range snap.Hosts {
		rows = append(rows, patrolHostAlertRow{
			id:       host.ID,
			name:     host.DisplayName,
			hostname: host.Hostname,
			status:   host.Status,
			platform: host.Platform,
			cpu:      host.CPUUsage,
			memory:   host.Memory.Usage,
		})
	}
	return rows
}

func patrolHostResourceType(platform string) string {
	return "agent"
}

func patrolLookupHostAlertResourceState(alert AlertInfo, snap patrolRuntimeState) (patrolAlertResourceState, bool) {
	for _, host := range patrolHostAlertRows(snap) {
		if !patrolAlertNameMatches(alert, host.id, host.name, host.hostname) {
			continue
		}
		name := host.hostname
		if name == "" {
			name = host.name
		}
		return patrolAlertResourceState{
			resourceType: patrolHostResourceType(host.platform),
			platform:     host.platform,
			name:         name,
			status:       host.status,
			cpu:          host.cpu,
			memory:       host.memory,
			found:        true,
		}, true
	}
	return patrolAlertResourceState{}, false
}

func (p *PatrolService) getCurrentMetricValueState(alert AlertInfo, snap patrolRuntimeState) float64 {
	resource := lookupPatrolAlertResourceState(alert, snap)
	if !resource.found {
		return -1
	}
	switch alert.Type {
	case "cpu":
		return resource.cpu
	case "memory":
		if resource.guestMemory != nil && !resource.guestMemory.pressureKnown {
			return -1
		}
		return resource.memory
	default:
		if resource.resourceType == "storage" {
			return resource.disk
		}
	}
	return -1
}

// isResourceOnline checks if a resource that triggered an offline alert is now online
func (p *PatrolService) isResourceOnlineState(alert AlertInfo, snap patrolRuntimeState) bool {
	resource := lookupPatrolAlertResourceState(alert, snap)
	if !resource.found {
		return false
	}
	switch resource.resourceType {
	case "node", "agent":
		return resource.status == "online"
	case "vm", "system-container", "app-container":
		return resource.status == "running" || resource.status == string(unifiedresources.StatusOnline)
	default:
		return false
	}
}

// patrolAlertReviewBatchSize caps how many alerts are reviewed per model
// call so a large alert backlog cannot build an unbounded prompt.
const patrolAlertReviewBatchSize = 20

type patrolAlertReviewVerdict struct {
	alert   AlertInfo
	resolve bool
	reason  string
}

// patrolAlertVerdictLine matches one per-alert verdict line in a batched
// alert-review model response, e.g. "ALERT 2: RESOLVE: usage back to 40%".
var patrolAlertVerdictLine = regexp.MustCompile(`(?im)^\s*ALERT\s+(\d+)\s*:\s*(RESOLVE|KEEP)\b\s*:?\s*(.*)$`)

// reviewAlertsWithModelState reviews the candidate alerts with the model in
// batched calls and returns one verdict per alert, in order. Alerts whose
// verdict cannot be parsed from the model response default to KEEP.
func (p *PatrolService) reviewAlertsWithModelState(ctx context.Context, candidates []AlertInfo, snap patrolRuntimeState, aiService *Service, executionID string) []patrolAlertReviewVerdict {
	verdicts := make([]patrolAlertReviewVerdict, 0, len(candidates))
	for start := 0; start < len(candidates); start += patrolAlertReviewBatchSize {
		end := min(start+patrolAlertReviewBatchSize, len(candidates))
		verdicts = append(verdicts, p.reviewAlertBatchState(ctx, candidates[start:end], snap, aiService, executionID)...)
	}
	return verdicts
}

// reviewAlertBatchState asks the model for a RESOLVE/KEEP verdict on each
// alert in one call. Any error or unparseable verdict keeps the alert —
// the fail-safe direction matches the single-alert behavior this replaces.
func (p *PatrolService) reviewAlertBatchState(ctx context.Context, batch []AlertInfo, snap patrolRuntimeState, aiService *Service, executionID string) []patrolAlertReviewVerdict {
	verdicts := make([]patrolAlertReviewVerdict, len(batch))
	for i, alert := range batch {
		verdicts[i] = patrolAlertReviewVerdict{alert: alert}
	}

	// Use a quick, low-cost AI call
	response, err := aiService.QuickAnalysis(ctx, QuickAnalysisRequest{
		Prompt:      p.buildAlertReviewPromptState(batch, snap),
		ExecutionID: executionID,
		UseCase:     "patrol",
		TargetType:  "alert_autoresolve",
	})
	if err != nil {
		log.Debug().Err(err).Int("alerts", len(batch)).Msg("AI Patrol: Failed to get AI judgment on alert batch")
		return verdicts
	}

	matched := applyAlertReviewResponse(verdicts, response)

	// Single-alert fallback: accept a bare "RESOLVE: <reason>" response
	// without the "ALERT 1:" prefix, matching the pre-batch prompt contract.
	if len(batch) == 1 && !matched {
		trimmed := strings.TrimSpace(response)
		if strings.HasPrefix(strings.ToUpper(trimmed), "RESOLVE:") {
			verdicts[0].resolve = true
			verdicts[0].reason = patrolAlertResolveReason(strings.TrimPrefix(trimmed[len("RESOLVE:"):], " "))
		}
	}
	// This quick review has only the supplied snapshot, not an independent
	// guest measurement. Missing/stale/cache-inclusive memory cannot prove a
	// memory alert recovered, even if the model asks to resolve it.
	for i, alert := range batch {
		// The snapshot carries no native sensor readings, so nothing in it
		// can show that one sensor's temperature recovered.
		if isNativeSensorAlertResource(alert.ResourceID) {
			verdicts[i].resolve = false
			verdicts[i].reason = ""
			continue
		}
		if alert.Type != "memory" {
			continue
		}
		resource := lookupPatrolAlertResourceState(alert, snap)
		if resource.guestMemory != nil && !resource.guestMemory.pressureKnown {
			verdicts[i].resolve = false
			verdicts[i].reason = ""
		}
	}

	return verdicts
}

// isNativeSensorAlertResource reports whether an alert names one native
// sensor of a host agent ("agent:<host>/sensor:<hex>").
func isNativeSensorAlertResource(resourceID string) bool {
	rest, ok := strings.CutPrefix(strings.TrimSpace(resourceID), "agent:")
	if !ok {
		return false
	}
	_, child, ok := strings.Cut(rest, "/")
	return ok && strings.HasPrefix(child, "sensor:")
}

// buildAlertReviewPromptState formats the batched alert-review prompt: one
// numbered section per alert with its trigger details and current resource
// state, plus strict per-line verdict output instructions. The current
// state covers only some alert conditions, so the prompt says that a
// condition it does not show is a KEEP, not a recovery.
func (p *PatrolService) buildAlertReviewPromptState(batch []AlertInfo, snap patrolRuntimeState) string {
	var sb strings.Builder
	sb.WriteString("Review these active alerts and determine, for each, if it should be auto-resolved based on current state.\n")
	for i, alert := range batch {
		alertType := patrolAlertLookupType(alert)
		sb.WriteString(fmt.Sprintf(`
ALERT %d:
- ID: %s
- Type: %s
- Resource: %s (%s)
- Message: %s
- Value when triggered: %.1f
- Threshold: %.1f
- Active for: %s

CURRENT STATE OF THIS RESOURCE:
%s
`,
			i+1, alert.ID, alert.Type, alert.ResourceName, alertType,
			alert.Message, alert.Value, alert.Threshold, alert.Duration,
			p.getResourceCurrentStateState(alert, snap)))
	}
	sb.WriteString(`
For each alert, should it be RESOLVED because the underlying issue is fixed?
Answer RESOLVE only when the current state shown for that alert directly shows its condition has cleared (for example the metric is back under its threshold, the resource is online again, or the reported health is healthy), or shows the resource no longer exists. If the current state does not cover the condition the alert is about, answer KEEP: normal readings for other metrics are not evidence that it is fixed.
Respond with EXACTLY one line per alert, in order, and nothing else:
ALERT <number>: RESOLVE: <brief reason>
or
ALERT <number>: KEEP: <brief reason>`)
	return sb.String()
}

// applyAlertReviewResponse parses per-alert verdict lines from the model
// response into verdicts. The first line seen for each alert number wins;
// out-of-range numbers are ignored. Returns whether any line matched.
func applyAlertReviewResponse(verdicts []patrolAlertReviewVerdict, response string) bool {
	matched := false
	seen := make(map[int]bool)
	for _, m := range patrolAlertVerdictLine.FindAllStringSubmatch(response, -1) {
		idx, err := strconv.Atoi(m[1])
		if err != nil || idx < 1 || idx > len(verdicts) || seen[idx] {
			continue
		}
		seen[idx] = true
		matched = true
		if strings.EqualFold(m[2], "RESOLVE") {
			verdicts[idx-1].resolve = true
			verdicts[idx-1].reason = patrolAlertResolveReason(m[3])
		}
	}
	return matched
}

func patrolAlertResolveReason(raw string) string {
	reason := strings.TrimSpace(raw)
	if reason == "" {
		reason = "issue no longer detected"
	}
	return "Patrol: " + reason
}

// getResourceCurrentState returns a description of the resource's current state
func (p *PatrolService) getResourceCurrentStateState(alert AlertInfo, snap patrolRuntimeState) string {
	resource := lookupPatrolAlertResourceState(alert, snap)
	if !resource.found {
		// The lookup reads only some alert subjects. One the canonical
		// inventory still holds, or whose owner it holds (a ZFS pool's
		// storage, a guest's backup, a Swarm service), is not missing, and
		// calling it missing invites the model to resolve it as removed.
		if patrolAlertSubjectInInventory(alert, snap) {
			return "Present in current inventory (this alert's resource or the resource that owns it); this review has no current reading of the alert's condition"
		}
		switch patrolAlertLookupType(alert) {
		case "storage":
			return "Storage not found in current state (may have been removed)"
		case "node":
			return "Node not found in current state"
		case "agent":
			return "Agent host not found in current state"
		case "vm":
			return "VM not found in current state"
		case "system-container":
			return "Container not found in current state"
		case "app-container":
			return "Docker container not found in current state"
		default:
			return "Resource state unknown"
		}
	}
	switch resource.resourceType {
	case "storage":
		return fmt.Sprintf("Storage '%s': %.1f%% used, status: %s", resource.name, resource.disk, resource.status)
	case "node":
		return fmt.Sprintf("Node '%s': CPU %.1f%%, Memory %.1f%%, Status: %s",
			resource.name, resource.cpu, resource.memory, resource.status)
	case "agent":
		if strings.EqualFold(strings.TrimSpace(resource.platform), "truenas") {
			return fmt.Sprintf("TrueNAS system '%s': CPU %.1f%%, Memory %.1f%%, Status: %s",
				resource.name, resource.cpu, resource.memory, resource.status)
		}
		return fmt.Sprintf("Agent host '%s': CPU %.1f%%, Memory %.1f%%, Status: %s",
			resource.name, resource.cpu, resource.memory, resource.status)
	case "vm":
		return fmt.Sprintf("VM '%s': CPU %.1f%%, Memory %s, Status: %s",
			resource.name, resource.cpu, resource.guestMemory.display(), resource.status)
	case "system-container":
		return fmt.Sprintf("Container '%s': CPU %.1f%%, Memory %s, Status: %s",
			resource.name, resource.cpu, resource.guestMemory.display(), resource.status)
	case "app-container":
		return fmt.Sprintf("Docker container '%s': CPU %.1f%%, Memory %.1f%%, State: %s, Health: %s, Restart count: %d, OOM-killed: %s",
			resource.name, resource.cpu, resource.memory, resource.status,
			dockerContainerHealthText(resource.health), resource.restartCount, dockerContainerOOMText(resource.oomKilled))
	default:
		return "Resource state unknown"
	}
}

// dockerContainerHealthText names the container's reported health check
// result, so a health alert's review sees the reading that raised it.
func dockerContainerHealthText(health string) string {
	if health = strings.TrimSpace(health); health == "" {
		return "none"
	}
	return health
}

// dockerContainerOOMText keeps an explicit "no" apart from an agent that
// does not report OOM kills, which proves nothing either way.
func dockerContainerOOMText(oomKilled *bool) string {
	switch {
	case oomKilled == nil:
		return "unknown"
	case *oomKilled:
		return "yes"
	default:
		return "no"
	}
}

// TriggerPatrolForAlert triggers an immediate patrol for a specific alert
func (p *PatrolService) TriggerPatrolForAlert(alert *alerts.Alert) {
	if alert == nil {
		return
	}

	p.mu.RLock()
	triggerManager := p.triggerManager
	eventTriggerConfig := p.eventTriggerConfig
	eventTriggerBlock := p.eventTriggerBlock
	p.mu.RUnlock()

	// Gate: skip if alert-driven scoped patrols are disabled
	if !eventTriggerConfig.AlertTriggersEnabled {
		log.Debug().
			Str("alert_identifier", alert.ID).
			Msg("alert-triggered patrol skipped: alert trigger source disabled")
		return
	}
	if eventTriggerBlock.active() {
		log.Debug().
			Str("alert_identifier", alert.ID).
			Str("block_reason", eventTriggerBlock.Reason).
			Msg("alert-triggered patrol skipped: event triggers blocked by runtime policy")
		return
	}

	resourceType := inferResourceType(alert.Type, alert.Metadata)

	if triggerManager != nil {
		scope := AlertTriggeredPatrolScope(alert.ID, alert.ResourceID, resourceType, alert.Type)
		if triggerManager.TriggerPatrol(scope) {
			log.Debug().Str("alert_identifier", alert.ID).Msg("queued alert-triggered patrol via trigger manager")
		} else {
			log.Warn().Str("alert_identifier", alert.ID).Msg("alert-triggered patrol rejected by trigger manager")
		}
		return
	}

	// Non-blocking send
	select {
	case p.adHocTrigger <- alert:
		log.Debug().Str("alert_identifier", alert.ID).Msg("queued ad-hoc patrol trigger")
	default:
		log.Warn().Str("alert_identifier", alert.ID).Msg("patrol trigger queue full, dropping trigger")
	}
}

func (p *PatrolService) beginRun(kind string) (*patrolRunStart, bool) {
	p.mu.Lock()
	if p.runInProgress {
		// Detect stuck runs: if the current run has been going for >20 minutes,
		// force-clear the flag so a new run can proceed.
		if !p.runStartedAt.IsZero() && time.Since(p.runStartedAt) > 20*time.Minute {
			log.Warn().
				Str("kind", kind).
				Time("started_at", p.runStartedAt).
				Dur("elapsed", time.Since(p.runStartedAt)).
				Msg("AI Patrol: Previous run appears stuck (>20min), force-clearing runInProgress")
			p.runInProgress = false
			// Fall through to start new run
		} else {
			p.mu.Unlock()
			if kind == "scoped" {
				GetPatrolMetrics().RecordScopedDropped()
				log.Warn().Str("kind", kind).Msg("AI Patrol: Run already in progress, dropping scoped patrol")
			} else {
				log.Debug().Str("kind", kind).Msg("AI Patrol: Run already in progress, skipping")
			}
			return nil, false
		}
	}
	startedAt := time.Now()
	runID := fmt.Sprintf("%d", startedAt.UnixNano())
	p.runInProgress = true
	p.currentRunID = runID
	p.runStartedAt = startedAt
	p.mu.Unlock()
	return &patrolRunStart{id: runID, startedAt: startedAt}, true
}

func (p *PatrolService) tryStartRun(kind string) bool {
	_, accepted := p.beginRun(kind)
	return accepted
}

func (p *PatrolService) endRun() {
	p.mu.Lock()
	p.runInProgress = false
	p.currentRunID = ""
	p.runStartedAt = time.Time{}
	orch := p.investigationOrchestrator
	p.mu.Unlock()

	// Periodic investigation store maintenance after each run
	if maintainer, ok := orch.(InvestigationStoreMaintainer); ok {
		maintainer.CleanupInvestigationStore(24*time.Hour, 1000)
	}
}

// runTargetedPatrol executes a focused patrol for a specific alert
func (p *PatrolService) runTargetedPatrol(ctx context.Context, alert *alerts.Alert) {
	log.Info().
		Str("alert_identifier", alert.ID).
		Str("resource_id", alert.ResourceID).
		Msg("Running targeted AI patrol for alert")

	resourceType := inferResourceType(alert.Type, alert.Metadata)
	scope := AlertTriggeredPatrolScope(alert.ID, alert.ResourceID, resourceType, alert.Type)
	p.TriggerScopedPatrol(ctx, scope)
}

// joinParts joins string parts with commas and "and" for the last element
func joinParts(parts []string) string {
	if len(parts) == 0 {
		return ""
	}
	if len(parts) == 1 {
		return parts[0]
	}
	if len(parts) == 2 {
		return parts[0] + " and " + parts[1]
	}
	return strings.Join(parts[:len(parts)-1], ", ") + ", and " + parts[len(parts)-1]
}

// generateFindingID creates a stable ID for a finding based on resource, category, and issue.
// All three components are included to ensure distinct issues on the same resource remain separate.
func generateFindingID(resourceID, category, issue string) string {
	hash := sha256.Sum256([]byte(fmt.Sprintf("%s:%s:%s", resourceID, category, issue)))
	return fmt.Sprintf("%x", hash[:8])
}