Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
// patrol_run.go implements the PatrolService runtime: Start/Stop lifecycle,
// the main patrol loop, scoped patrol execution, alert auto-resolution,
// live streaming to UI subscribers, and run history tracking.
package ai
import (
"context"
"crypto/sha256"
"errors"
"fmt"
"regexp"
"strconv"
"strings"
"time"
"github.com/google/uuid"
"github.com/rcourtman/pulse-go-rewrite/internal/ai/circuit"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
"github.com/rs/zerolog/log"
)
// Patrol run lifecycle constants.
const (
initialPatrolStartDelay = 30 * time.Second // Delay before first patrol after startup
findingCleanupAge = 24 * time.Hour // Resolved findings older than this are purged
scopedPatrolRetryBackoff1 = 5 * time.Second // First retry backoff for dropped scoped patrols
scopedPatrolRetryBackoff2 = 15 * time.Second // Second retry backoff for dropped scoped patrols
scopedPatrolMaxRetries = 2 // Maximum re-queue attempts for dropped scoped patrols
scopedPatrolLogIDLimit = 10 // Maximum number of effective scope IDs to log inline
remindSweepInterval = time.Hour // Cadence for the proactive will_fix_later remind sweep
remindSweepInitialDelay = 30 * time.Second // First sweep runs shortly after start so a restart-with-overdue does not have to wait a full hour
)
// Start begins the background patrol loop
func (p *PatrolService) Start(ctx context.Context) {
p.mu.Lock()
if p.running {
p.mu.Unlock()
return
}
p.running = true
p.stopCh = make(chan struct{})
p.configChanged = make(chan struct{}, 1) // Buffered to allow non-blocking send
p.mu.Unlock()
log.Info().
Dur("interval", p.config.GetInterval()).
Msg("Starting AI Patrol Service")
go p.patrolLoop(ctx)
go p.remindSweepLoop(ctx)
go p.objectiveObserverLoop(ctx)
}
// Stop stops the patrol service. It signals the patrol loop to exit, then
// waits up to 15 seconds for in-flight investigations to finish and
// force-saves findings/investigation state to disk.
func (p *PatrolService) Stop() {
p.mu.Lock()
if !p.running {
p.mu.Unlock()
return
}
p.running = false
close(p.stopCh)
orchestrator := p.investigationOrchestrator
findings := p.findings
p.mu.Unlock()
log.Info().Msg("stopping AI Patrol Service")
// Give investigations 15 seconds to finish (leaves headroom within server's 30s budget)
shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 15*time.Second)
defer shutdownCancel()
// Signal orchestrator to cancel running investigations and persist state
if orchestrator != nil {
if err := orchestrator.Shutdown(shutdownCtx); err != nil {
log.Warn().Err(err).Msg("AI Patrol: Investigation orchestrator shutdown returned error")
}
}
// Wait for investigation goroutines tracked by PatrolService
done := make(chan struct{})
go func() {
p.investigationWg.Wait()
close(done)
}()
select {
case <-done:
// All investigation goroutines finished
case <-shutdownCtx.Done():
log.Warn().Msg("AI Patrol: Timed out waiting for investigation goroutines to finish")
}
// Force-save findings store
if findings != nil {
if err := findings.ForceSave(); err != nil {
log.Error().Err(err).Msg("AI Patrol: Failed to force-save findings during shutdown")
}
}
log.Info().Msg("AI Patrol Service stopped")
}
// patrolLoop is the main background loop
func (p *PatrolService) patrolLoop(ctx context.Context) {
// Seed recency from persisted run history so the API can return Patrol timing
// metadata immediately (before the first in-process patrol completes).
if history := p.GetRunHistory(10); len(history) > 0 {
lastActivity, lastFullPatrol := patrolRecencyFromHistory(history)
p.mu.Lock()
p.lastActivity = lastActivity
p.lastFullPatrol = lastFullPatrol
p.mu.Unlock()
}
// Let configuration and collection initialize before resuming a due check.
// This grace period must not become a new full scheduling interval.
initialTimer := time.NewTimer(initialPatrolStartDelay)
defer initialTimer.Stop()
select {
case <-initialTimer.C:
case <-p.stopCh:
if !initialTimer.Stop() {
select {
case <-initialTimer.C:
default:
}
}
return
case <-ctx.Done():
if !initialTimer.Stop() {
select {
case <-initialTimer.C:
default:
}
}
return
}
p.mu.RLock()
interval := p.config.GetInterval()
configCh := p.configChanged
p.mu.RUnlock()
next, trigger := p.initialPatrolSchedule(time.Now(), interval)
if trigger != TriggerReasonStartup {
p.setNextPatrolSchedule(next, interval)
}
timer := time.NewTimer(time.Until(next))
defer timer.Stop()
for {
select {
case <-timer.C:
if trigger == TriggerReasonStartup {
p.runPatrolWithTrigger(ctx, trigger, nil)
// Keep the first-install cadence anchored after its startup check.
next = time.Now().Add(interval)
} else {
// Persist the next deadline before starting work. Collapse missed
// intervals into one due check rather than replaying a backlog.
next = nextPatrolDeadline(next, time.Now(), interval)
p.setNextPatrolSchedule(next, interval)
p.runPatrolWithTrigger(ctx, trigger, nil)
}
trigger = TriggerReasonScheduled
p.setNextPatrolSchedule(next, interval)
timer.Reset(time.Until(next))
case alert := <-p.adHocTrigger:
// Run immediate targeted patrol for this alert
log.Info().Str("alert_identifier", alert.ID).Msg("patrol triggered by alert")
p.runTargetedPatrol(ctx, alert)
case <-configCh:
// An operator interval change deliberately resets the timer. Persist
// that exact reset too, even if no new full run has completed yet.
p.mu.RLock()
newInterval := p.config.GetInterval()
p.mu.RUnlock()
if newInterval != interval {
interval = newInterval
if trigger != TriggerReasonStartup {
next = time.Now().Add(interval)
p.setNextPatrolSchedule(next, interval)
timer.Reset(time.Until(next))
}
log.Info().
Dur("interval", interval).
Msg("Patrol schedule reset to new interval")
}
case <-p.stopCh:
return
case <-ctx.Done():
return
}
}
}
// remindSweepLoop fires SweepWillFixLaterReminders on a 1h cadence so
// will_fix_later commitments past their RemindAt deadline surface even
// when patrol detection is sparse or the operational issue transiently
// clears between runs. Runs an initial sweep ~30s after start so a
// restart-with-overdue does not have to wait a full hour.
func (p *PatrolService) remindSweepLoop(ctx context.Context) {
sweep := func() {
if p.findings == nil {
return
}
swept := p.findings.SweepWillFixLaterReminders(time.Now())
if swept > 0 {
log.Info().Int("swept", swept).Msg("Patrol remind-sweep promoted overdue will_fix_later commitments")
}
}
initialTimer := time.NewTimer(remindSweepInitialDelay)
defer initialTimer.Stop()
select {
case <-initialTimer.C:
sweep()
case <-p.stopCh:
return
case <-ctx.Done():
return
}
ticker := time.NewTicker(remindSweepInterval)
defer ticker.Stop()
for {
select {
case <-ticker.C:
sweep()
case <-p.stopCh:
return
case <-ctx.Done():
return
}
}
}
func (p *PatrolService) initialPatrolSchedule(now time.Time, interval time.Duration) (time.Time, TriggerReason) {
if p.runHistoryStore != nil {
if next, savedInterval := p.runHistoryStore.Schedule(); !next.IsZero() && savedInterval == interval {
return next, TriggerReasonScheduled
}
}
// Upgrade old history without a timer cursor. Use all retained full runs:
// recent scoped/verification activity must not hide or move the cadence.
if last := p.LastSuccessfulFullPatrolAt(); !last.IsZero() {
return last.Add(interval), TriggerReasonScheduled
}
return now, TriggerReasonStartup
}
func nextPatrolDeadline(due, now time.Time, interval time.Duration) time.Time {
next := due.Add(interval)
if !next.After(now) {
return now.Add(interval)
}
return next
}
func (p *PatrolService) setNextPatrolSchedule(next time.Time, interval time.Duration) {
p.mu.Lock()
p.nextScheduledAt = next
enabled, store := p.config.Enabled, p.runHistoryStore
p.mu.Unlock()
// Disabled timer ticks are not checks. In particular, first-time provider
// setup must not leave a durable cursor that suppresses the startup check.
if enabled && store != nil {
if err := store.SetSchedule(next, interval); err != nil {
log.Error().Err(err).Time("next_scheduled_at", next).Msg("Failed to persist Patrol schedule")
}
}
}
// LastSuccessfulFullPatrolAt returns the completion time of the most recent
// full patrol run that finished without errors, or the zero time when none
// exists. The manual-run cadence gate uses this rather than lastFullPatrol so
// a failed run does not consume the Community hourly slot.
func (p *PatrolService) LastSuccessfulFullPatrolAt() time.Time {
if p == nil || p.runHistoryStore == nil {
return time.Time{}
}
var last time.Time
for _, run := range p.runHistoryStore.GetAll() {
if run.CompletedAt.IsZero() || !isSuccessfulFullPatrolRun(run) {
continue
}
if run.CompletedAt.After(last) {
last = run.CompletedAt
}
}
return last
}
func patrolRecencyFromHistory(runHistory []PatrolRunRecord) (time.Time, time.Time) {
var lastActivity time.Time
var lastFullPatrol time.Time
for _, run := range runHistory {
if run.CompletedAt.IsZero() {
continue
}
if lastActivity.IsZero() || run.CompletedAt.After(lastActivity) {
lastActivity = run.CompletedAt
}
if isFullPatrolRun(run) && (lastFullPatrol.IsZero() || run.CompletedAt.After(lastFullPatrol)) {
lastFullPatrol = run.CompletedAt
}
}
return lastActivity, lastFullPatrol
}
// runPatrol executes a scheduled patrol run
func (p *PatrolService) runPatrol(ctx context.Context) {
p.runPatrolWithTrigger(ctx, TriggerReasonScheduled, nil)
}
type patrolRunStart struct {
id string
startedAt time.Time
}
func (p *PatrolService) recordAcceptedRunFailure(start *patrolRunStart, trigger TriggerReason, summary, detail string) {
if start == nil || p.runHistoryStore == nil {
return
}
completedAt := time.Now()
duration := completedAt.Sub(start.startedAt)
p.runHistoryStore.Add(PatrolRunRecord{
ID: start.id,
StartedAt: start.startedAt,
CompletedAt: completedAt,
Duration: duration,
DurationMs: duration.Milliseconds(),
Type: "patrol",
TriggerReason: string(trigger),
FindingsSummary: "Analysis incomplete (1 error)",
FindingIDs: []string{},
ErrorCount: 1,
Status: "error",
ErrorSummary: summary,
ErrorDetail: detail,
})
p.mu.Lock()
p.lastActivity = completedAt
p.lastFullPatrol = completedAt
p.lastDuration = duration
p.resourcesChecked = 0
p.errorCount = 1
p.mu.Unlock()
}
// runPatrolWithTrigger executes a patrol run with trigger context
func (p *PatrolService) runPatrolWithTrigger(ctx context.Context, trigger TriggerReason, scope *PatrolScope) {
p.runPatrolWithTriggerStart(ctx, trigger, scope, nil)
}
// runPatrolWithTriggerStart executes a patrol run, optionally using a start
// reservation that was accepted synchronously by the manual-run API. Keeping
// acceptance and execution on the same reservation removes the interval where
// the API has returned success but status still reports no active run.
func (p *PatrolService) runPatrolWithTriggerStart(ctx context.Context, trigger TriggerReason, scope *PatrolScope, acceptedStart *patrolRunStart) {
p.mu.RLock()
cfg := p.config
breaker := p.circuitBreaker
p.mu.RUnlock()
// Demo/mock mode simulates patrol passes without a provider. Checked live
// (not from the boot-time config snapshot) because release demo instances
// enable mock fixtures only after the license sync authorizes them.
if IsDemoMode() {
if !p.runDemoPatrolCycleWithStart(trigger, acceptedStart) && acceptedStart != nil {
p.recordAcceptedRunFailure(acceptedStart, trigger,
"Patrol demo state unavailable",
"Pulse accepted the run, but demo infrastructure state was not available.")
}
return
}
if !cfg.Enabled {
if acceptedStart != nil {
p.recordAcceptedRunFailure(acceptedStart, trigger,
"Patrol disabled before execution",
"Pulse accepted the run, but Patrol was disabled before execution began.")
p.endRun()
}
return
}
if reason := strings.TrimSpace(cfg.RuntimeBlockedReason); reason != "" {
if acceptedStart != nil {
p.recordAcceptedRunFailure(acceptedStart, trigger,
"Patrol runtime became unavailable",
reason)
p.endRun()
}
p.setBlockedReasonWithCause(reason, cfg.RuntimeBlockedCause)
p.raiseBlockedRunFinding(reason, cfg.RuntimeBlockedCause)
log.Info().Str("reason", reason).Str("cause", string(cfg.RuntimeBlockedCause)).Msg("AI Patrol: Skipping run - runtime readiness blocked")
return
}
runStart := acceptedStart
if runStart == nil {
var accepted bool
runStart, accepted = p.beginRun("full")
if !accepted {
return
}
}
defer p.endRun()
start := runStart.startedAt
runID := runStart.id
executionID := uuid.NewString()
patrolType := "patrol"
GetPatrolMetrics().RecordRun(string(trigger), "full")
log.Debug().Msg("AI Patrol: Starting patrol run")
// Track run statistics
var runStats struct {
resourceCount int
nodesChecked int
guestsChecked int
dockerChecked int
storageChecked int
hostsChecked int
trueNASChecked int
pbsChecked int
pmgChecked int
kubernetesChecked int
newFindings int
existingFindings int
rejectedFindings int
triageFlags int
triageSkippedLLM bool
findingIDs []string
errors int
lastAIError error // Preserve original error for circuit breaker categorization
errorSummary string
errorDetail string
aiAnalysis *AIAnalysisResult // Stores the AI's analysis for the run record
runtimeResolved bool
}
var newFindings []*Finding
// Get current state
if !p.hasPatrolRuntimeInputs() {
log.Warn().Msg("AI Patrol: No runtime state available")
if acceptedStart != nil {
p.recordAcceptedRunFailure(runStart, trigger,
"Patrol runtime state unavailable",
"Pulse accepted the run, but no infrastructure state was available for analysis.")
}
return
}
state := p.currentPatrolRuntimeState()
// Helper to track findings
// Note: Only warning+ severity findings count toward newFindings since watch/info are filtered from UI
trackFinding := func(f *Finding) bool {
isNew := p.recordFinding(f)
if isNew {
// Only count warning+ findings as "new" for user-facing stats
if f.Severity == FindingSeverityWarning || f.Severity == FindingSeverityCritical {
runStats.newFindings++
newFindings = append(newFindings, f)
}
} else {
runStats.existingFindings++
}
// Only track warning+ severity finding IDs in the run record
if f.Severity == FindingSeverityWarning || f.Severity == FindingSeverityCritical {
runStats.findingIDs = append(runStats.findingIDs, f.ID)
}
return isNew
}
// Count resources for statistics from the patrol runtime state so scoped and
// unscoped runs share the same read-state-first semantics.
resourceCounts := patrolRuntimeCountResources(state)
if cfg.AnalyzeNodes {
runStats.nodesChecked = resourceCounts.nodes
}
if cfg.AnalyzeGuests {
runStats.guestsChecked = resourceCounts.guests
}
if cfg.AnalyzeDocker {
runStats.dockerChecked = resourceCounts.docker
}
if cfg.AnalyzeStorage {
runStats.storageChecked = resourceCounts.storage
}
if cfg.AnalyzePBS {
runStats.pbsChecked = resourceCounts.pbs
}
if cfg.AnalyzePMG {
runStats.pmgChecked = resourceCounts.pmg
}
if cfg.AnalyzeHosts {
runStats.hostsChecked = resourceCounts.hosts
runStats.trueNASChecked = resourceCounts.truenas
}
if cfg.AnalyzeKubernetes {
runStats.kubernetesChecked = resourceCounts.kubernetes
}
runStats.resourceCount = runStats.nodesChecked + runStats.guestsChecked +
runStats.dockerChecked + runStats.storageChecked + runStats.pbsChecked + runStats.pmgChecked + runStats.hostsChecked +
runStats.trueNASChecked +
runStats.kubernetesChecked
// Run update-safety watcher on every patrol cycle regardless of LLM availability.
// Direct FindingsStore.Add bypasses push-notification (MVP: not an emergency).
if cfg.AnalyzeDocker && p.updateSafetyWatcher != nil {
updateEmit, updateResolve := p.updateSafetyWatcher.Observe(patrolUpdateSafetyDockerHosts(state), time.Now())
for _, f := range updateEmit {
if !p.findings.Add(f) {
log.Debug().Str("finding_id", f.ID).Msg("AI Patrol: update-safety finding already in store")
}
}
for _, s := range updateResolve {
p.findings.ResolveWithReason(s.DedupKey, s.Reason)
}
}
if cfg.AnalyzeHosts && p.aptWorkflowWatcher != nil {
aptEmit, aptResolve := p.aptWorkflowWatcher.Observe(state, p.findings.GetActive(FindingSeverityInfo), time.Now())
for _, f := range aptEmit {
trackFinding(f)
}
for _, sentinel := range aptResolve {
if p.findings.ResolveWithReason(sentinel.DedupKey, sentinel.Reason) {
p.mu.RLock()
resolveUnified := p.unifiedFindingResolver
p.mu.RUnlock()
if resolveUnified != nil {
resolveUnified(sentinel.DedupKey)
}
}
}
}
if cfg.AnalyzeGuests && p.proxmoxGuestLifecycleWatcher != nil {
proxmoxEmit, proxmoxResolve := p.proxmoxGuestLifecycleWatcher.Observe(state, p.findings.GetActive(FindingSeverityInfo), time.Now())
for _, f := range proxmoxEmit {
trackFinding(f)
}
for _, sentinel := range proxmoxResolve {
if p.findings.ResolveWithReason(sentinel.DedupKey, sentinel.Reason) {
p.mu.RLock()
resolveUnified := p.unifiedFindingResolver
p.mu.RUnlock()
if resolveUnified != nil {
resolveUnified(sentinel.DedupKey)
}
}
}
}
// Run PDM alert bridge on every patrol cycle if a source is configured.
// Nil source guard mirrors the MVP no-op behavior; the real HTTP client
// is wired in a follow-on. Direct FindingsStore.Add bypasses push.
if p.pdmAlertBridge != nil && p.pdmAlertBridge.source != nil {
pdmEmit, pdmResolve := p.pdmAlertBridge.Observe(ctx, time.Now())
for _, f := range pdmEmit {
if !p.findings.Add(f) {
log.Debug().Str("finding_id", f.ID).Msg("AI Patrol: PDM alert finding already in store")
}
}
for _, s := range pdmResolve {
p.findings.ResolveWithReason(s.DedupKey, s.Reason)
}
}
// Acquire a breaker probe only when this run is actually ready to call the
// provider. Early collection/scope exits must not strand a half-open probe.
aiServiceEnabled := p.retryProviderInitIfNeeded(ctx)
llmAllowed := true
if aiServiceEnabled && breaker != nil {
llmAllowed = breaker.Allow()
}
if !llmAllowed {
log.Warn().Msg("AI Patrol: Circuit breaker is open (LLM calls blocked)")
}
canRunLLM := aiServiceEnabled && llmAllowed
// Check if we can run LLM analysis (AI-only patrol)
if !canRunLLM {
reason, cause := p.providerUnavailableReason()
if aiServiceEnabled && !llmAllowed {
reason = "circuit breaker is open"
cause = PatrolFailureCauseCircuitOpen
GetPatrolMetrics().RecordCircuitBlock()
} else {
// A run skipped for a persistent provider state (never attempted)
// surfaces as the deduped runtime finding; a transiently open
// breaker does not, because the attempts that opened it already
// raised their own specific finding.
trackFinding(newPatrolRuntimeFailureFinding(patrolBlockedRunFailure(reason, cause), time.Now()))
}
p.setBlockedReasonWithCause(reason, cause)
log.Info().Str("reason", reason).Str("cause", string(cause)).Msg("AI Patrol: Skipping run - AI unavailable")
runStats.errors++
runStats.errorSummary = "Patrol provider unavailable"
runStats.errorDetail = reason
} else {
p.clearBlockedReason()
// Ensure stream state is clean for this run before the first streamed event.
p.resetStreamForRun(runID)
// Run agentic AI analysis — the LLM uses tools to investigate and reports findings
aiResult, aiErr := p.runAIAnalysisState(ctx, state, scope, executionID)
if aiResult != nil {
runStats.aiAnalysis = aiResult
runStats.rejectedFindings = aiResult.RejectedFindings
runStats.triageFlags = aiResult.TriageFlags
runStats.triageSkippedLLM = aiResult.TriageSkippedLLM
// Findings are recorded by the Patrol finding tools before the model
// response completes. Preserve their run-level accounting even when a
// later provider error interrupts the same analysis.
for _, f := range aiResult.Findings {
if f.Severity == FindingSeverityWarning || f.Severity == FindingSeverityCritical {
runStats.findingIDs = append(runStats.findingIDs, f.ID)
stored := p.findings.Get(f.ID)
if stored != nil && stored.TimesRaised <= 1 {
runStats.newFindings++
newFindings = append(newFindings, f)
} else {
runStats.existingFindings++
}
}
}
}
if aiErr != nil {
log.Warn().Err(aiErr).Msg("AI Patrol: LLM analysis failed")
runStats.errors++
runStats.lastAIError = aiErr
failure := patrolRuntimeFailureFromError(aiErr)
runStats.errorSummary = failure.Summary
runStats.errorDetail = failure.Detail
p.blockOnExhaustedBudget(aiErr, failure)
// Create a finding to surface this error to the user
errorFinding := newPatrolRuntimeFailureFinding(failure, time.Now())
trackFinding(errorFinding)
} else if aiResult != nil {
if missing := patrolMissingAssessmentIDs(aiResult); len(missing) > 0 {
runStats.errors++
runStats.errorSummary = "Patrol finding assessment incomplete"
runStats.errorDetail = fmt.Sprintf("No explicit verdict was accepted for active finding IDs: %s", strings.Join(missing, ", "))
}
if !aiResult.TriageSkippedLLM {
runStats.runtimeResolved = p.resolvePatrolRuntimeFailureFinding("full_patrol_success")
}
// Attach the deterministic capacity forecasts computed this run to
// matching findings, so the surface shows a first-class urgency
// signal (trend/eta) instead of relying on model prose.
p.stampCapacityForecasts(aiResult.Forecasts)
}
}
// Fold findings that restate an active alert under that alert. Runs after
// both the deterministic watchers and the model have written this cycle's
// findings, against the unscoped alert set, so scoped runs cannot release
// mirrors outside their scope.
if mirrored := p.reconcileAlertMirrors(); mirrored > 0 {
log.Debug().Int("changed", mirrored).Msg("AI Patrol: Updated alert-mirror stamps on findings")
}
// Count resolved findings: LLM-resolved (via tool) + auto-reconciled stale findings.
var resolvedCount int
if runStats.aiAnalysis != nil {
resolvedCount = len(runStats.aiAnalysis.ResolvedIDs)
if runStats.runtimeResolved {
resolvedCount++
}
// Auto-resolve stale findings: active findings that were presented to the LLM
// in seed context but were neither re-reported nor explicitly resolved.
// Only runs after successful full patrols (not scoped).
observedIDs := append(append([]string{}, runStats.aiAnalysis.ReportedIDs...), assessmentFindingIDs(runStats.aiAnalysis.Assessments)...)
autoResolved := p.reconcileStaleFindings(
observedIDs,
runStats.aiAnalysis.ResolvedIDs,
runStats.aiAnalysis.SeededFindingIDs,
runStats.errors > 0,
)
resolvedCount += autoResolved
if autoResolved > 0 {
log.Info().
Int("auto_resolved", autoResolved).
Msg("AI Patrol: Auto-resolved stale findings after full patrol")
}
}
// Cleanup old resolved findings (always runs, doesn't require LLM)
cleaned := p.findings.Cleanup(findingCleanupAge)
if cleaned > 0 {
log.Debug().Int("cleaned", cleaned).Msg("AI Patrol: Cleaned up old findings")
}
// Recover investigations stuck in "running" state (goroutine panicked or was killed)
p.recoverStuckInvestigations()
// Retry investigations that failed due to timeout (shorter cooldown than permanent failures)
p.retryTimedOutInvestigations()
// AI-based alert review: check active alerts against current state and auto-resolve fixed issues
// Pass llmAllowed so it knows whether AI calls are allowed.
alertsResolved := p.reviewAndResolveAlertsState(ctx, state, llmAllowed, executionID)
if alertsResolved > 0 {
log.Info().Int("alerts_resolved", alertsResolved).Msg("AI Patrol: Auto-resolved alerts where issues are fixed")
}
duration := time.Since(start)
completedAt := time.Now()
// Build findings summary string.
summary := p.findings.GetSummary()
var findingsSummaryStr string
var status string
// Only count critical and warning as active issues (watch/info are filtered from UI)
totalActive := summary.Critical + summary.Warning
if totalActive == 0 {
findingsSummaryStr = "All healthy"
status = "healthy"
} else {
parts := []string{}
if summary.Critical > 0 {
parts = append(parts, fmt.Sprintf("%d critical", summary.Critical))
}
if summary.Warning > 0 {
parts = append(parts, fmt.Sprintf("%d warning", summary.Warning))
}
findingsSummaryStr = fmt.Sprintf("%s", joinParts(parts))
if summary.Critical > 0 {
status = "critical"
} else {
status = "issues_found"
}
}
if runStats.errors > 0 {
status = "error"
// Don't claim "All healthy" if there were errors - the patrol didn't complete properly
if findingsSummaryStr == "All healthy" {
findingsSummaryStr = fmt.Sprintf("Analysis incomplete (%d errors)", runStats.errors)
}
}
// Create run record
runRecord := PatrolRunRecord{
ID: runID,
StartedAt: start,
CompletedAt: completedAt,
Duration: duration,
DurationMs: duration.Milliseconds(),
Type: patrolType,
TriggerReason: string(trigger),
ResourcesChecked: runStats.resourceCount,
NodesChecked: runStats.nodesChecked,
GuestsChecked: runStats.guestsChecked,
DockerChecked: runStats.dockerChecked,
StorageChecked: runStats.storageChecked,
HostsChecked: runStats.hostsChecked,
TrueNASChecked: runStats.trueNASChecked,
PBSChecked: runStats.pbsChecked,
PMGChecked: runStats.pmgChecked,
KubernetesChecked: runStats.kubernetesChecked,
NewFindings: runStats.newFindings,
ExistingFindings: runStats.existingFindings,
RejectedFindings: runStats.rejectedFindings,
ResolvedFindings: resolvedCount,
FindingsSummary: findingsSummaryStr,
FindingIDs: patrolRunFindingIDs(runStats.findingIDs, assessmentsForRun(runStats.aiAnalysis)),
ErrorCount: runStats.errors,
Status: status,
ErrorSummary: runStats.errorSummary,
ErrorDetail: runStats.errorDetail,
}
if scope != nil {
runRecord.ScopeResourceIDs = scope.ResourceIDs
runRecord.ScopeResourceTypes = scope.ResourceTypes
runRecord.ScopeContext = scope.Context
runRecord.ObjectiveContext = clonePatrolObjectiveContext(scope.ObjectiveContext)
runRecord.AlertIdentifier = scope.AlertIdentifier
runRecord.FindingID = scope.FindingID
}
// Add AI analysis details if available
if runStats.aiAnalysis != nil {
runRecord.AIAnalysis = patrolRunAIAnalysisForRecord(runStats.aiAnalysis, patrolRunAnalysisRecordContext{
ResourcesChecked: runStats.resourceCount,
NewFindings: runStats.newFindings,
ExistingFindings: runStats.existingFindings,
ResolvedFindings: resolvedCount,
UncertainFindings: patrolUncertainAssessmentCount(runStats.aiAnalysis.Assessments),
ErrorCount: runStats.errors,
})
runRecord.FindingAssessments = append([]PatrolFindingAssessment(nil), runStats.aiAnalysis.Assessments...)
runRecord.InputTokens = runStats.aiAnalysis.InputTokens
runRecord.OutputTokens = runStats.aiAnalysis.OutputTokens
runRecord.CacheCreationInputTokens = runStats.aiAnalysis.CacheCreationInputTokens
runRecord.CacheReadInputTokens = runStats.aiAnalysis.CacheReadInputTokens
runRecord.TriageFlags = runStats.triageFlags
runRecord.TriageSkippedLLM = runStats.triageSkippedLLM
toolCalls := runStats.aiAnalysis.ToolCalls
if len(toolCalls) > MaxToolCallsPerRun {
toolCalls = toolCalls[:MaxToolCallsPerRun]
}
runRecord.ToolCalls = toolCalls
runRecord.ToolCallCount = len(runStats.aiAnalysis.ToolCalls)
log.Debug().
Int("response_length", len(runStats.aiAnalysis.Response)).
Int("input_tokens", runStats.aiAnalysis.InputTokens).
Int("output_tokens", runStats.aiAnalysis.OutputTokens).
Int("tool_calls", runRecord.ToolCallCount).
Msg("AI Patrol: Storing AI analysis in run record")
} else {
log.Debug().Msg("AI Patrol: No AI analysis to store (aiAnalysis is nil)")
}
p.mu.Lock()
p.lastActivity = completedAt
p.lastFullPatrol = completedAt
p.lastDuration = duration
p.resourcesChecked = runStats.resourceCount
p.errorCount = runStats.errors
p.mu.Unlock()
// Record circuit breaker result only if we actually attempted LLM calls.
// canRunLLM is true only when AI is enabled, licensed, AND breaker allowed.
// Use error categorization so non-transient errors (auth failures, insufficient
// credits) don't trip the breaker — those won't be fixed by waiting.
recordPatrolCircuitResult(breaker, canRunLLM, runStats.errors, runStats.lastAIError)
// Add to history store (handles persistence automatically)
p.runHistoryStore.Add(runRecord)
log.Info().
Str("type", patrolType).
Dur("duration", duration).
Int("resources", runStats.resourceCount).
Int("new_findings", runStats.newFindings).
Int("resolved", resolvedCount).
Int("critical", summary.Critical).
Int("warning", summary.Warning).
Int("watch", summary.Watch).
Msg("AI Patrol: Completed patrol run")
p.dispatchPatrolInvestigations(runStats.aiAnalysis)
}
func patrolFindingSummaryForRun(findings []*Finding, findingIDs []string) FindingsSummary {
owned := make(map[string]bool, len(findingIDs))
for _, findingID := range findingIDs {
if findingID = strings.TrimSpace(findingID); findingID != "" {
owned[findingID] = true
}
}
var summary FindingsSummary
for _, finding := range findings {
if finding == nil || !owned[finding.ID] {
continue
}
summary.Total++
switch finding.Severity {
case FindingSeverityCritical:
summary.Critical++
case FindingSeverityWarning:
summary.Warning++
case FindingSeverityWatch:
summary.Watch++
case FindingSeverityInfo:
summary.Info++
}
}
return summary
}
// runScopedPatrol runs a patrol on a filtered subset of resources.
// This provides token-efficient analysis for event-driven patrols.
func (p *PatrolService) runScopedPatrol(ctx context.Context, scope PatrolScope) {
p.runScopedPatrolWithStart(ctx, scope, nil, true)
}
// runScopedPatrolWithStart executes a scoped run either from the background
// trigger queue or from a reservation accepted synchronously by the manual-run
// API. Background triggers retain their bounded retry behaviour. A manual
// reservation is already durable and therefore must either produce a run
// record or an explicit terminal failure; it is never silently re-queued.
func (p *PatrolService) runScopedPatrolWithStart(ctx context.Context, scope PatrolScope, acceptedStart *patrolRunStart, retryOnBusy bool) {
p.mu.RLock()
cfg := p.config
breaker := p.circuitBreaker
p.mu.RUnlock()
// Demo instances simulate scheduled patrol passes only; event-driven
// scoped runs would hit the real provider path.
if IsDemoMode() {
if acceptedStart != nil {
p.recordScopedPatrolScopeFailure(acceptedStart.startedAt, acceptedStart.id, scope,
PatrolScopeResolution{RequestedResourceIDs: append([]string(nil), scope.ResourceIDs...)},
"Scoped Patrol is unavailable in demo mode",
"Pulse accepted the scoped run, but the demo runtime cannot execute event-driven scoped checks.")
p.endRun()
}
return
}
if !cfg.Enabled {
if acceptedStart != nil {
p.recordScopedPatrolScopeFailure(acceptedStart.startedAt, acceptedStart.id, scope,
PatrolScopeResolution{RequestedResourceIDs: append([]string(nil), scope.ResourceIDs...)},
"Patrol disabled before scoped execution",
"Pulse accepted the scoped run, but Patrol was disabled before execution began.")
p.endRun()
}
return
}
if reason := strings.TrimSpace(cfg.RuntimeBlockedReason); reason != "" {
if acceptedStart != nil {
p.recordScopedPatrolScopeFailure(acceptedStart.startedAt, acceptedStart.id, scope,
PatrolScopeResolution{RequestedResourceIDs: append([]string(nil), scope.ResourceIDs...)},
"Patrol runtime became unavailable",
reason)
p.endRun()
}
p.setBlockedReasonWithCause(reason, cfg.RuntimeBlockedCause)
log.Info().Str("reason", reason).Str("cause", string(cfg.RuntimeBlockedCause)).Msg("AI Patrol: Skipping scoped run - runtime readiness blocked")
return
}
runStart := acceptedStart
if runStart == nil {
var accepted bool
runStart, accepted = p.beginRun("scoped")
// Re-queue background triggers with backoff if retries remain.
if !accepted && retryOnBusy && scope.RetryCount < scopedPatrolMaxRetries {
scope.RetryCount++
backoff := scopedPatrolRetryBackoff1
if scope.RetryCount == scopedPatrolMaxRetries {
backoff = scopedPatrolRetryBackoff2
}
scope.RetryAfter = time.Now().Add(backoff)
if tm := p.GetTriggerManager(); tm != nil {
tm.TriggerPatrol(scope)
log.Info().
Int("retry", scope.RetryCount).
Dur("backoff", backoff).
Strs("resources", scope.ResourceIDs).
Msg("AI Patrol: Re-queued dropped scoped patrol with backoff")
}
} else if !accepted && retryOnBusy {
GetPatrolMetrics().RecordScopedDroppedFinal()
log.Error().
Strs("resources", scope.ResourceIDs).
Str("reason", string(scope.Reason)).
Msg("AI Patrol: Scoped patrol permanently dropped after 2 retries")
}
if !accepted {
return
}
}
defer p.endRun()
start := runStart.startedAt
runID := runStart.id
executionID := uuid.NewString()
if scope.Reason == TriggerReasonObjectiveChanged && !p.beginObjectivePlanning(scope, start) {
return
}
GetPatrolMetrics().RecordRun(string(scope.Reason), "scoped")
var runStats struct {
resourceCount int
nodesChecked int
guestsChecked int
dockerChecked int
storageChecked int
hostsChecked int
trueNASChecked int
pbsChecked int
pmgChecked int
kubernetesChecked int
newFindings int
existingFindings int
rejectedFindings int
triageFlags int
triageSkippedLLM bool
findingIDs []string
errors int
lastAIError error
errorSummary string
errorDetail string
aiAnalysis *AIAnalysisResult
runtimeResolved bool
}
// Get current state
if !p.hasPatrolRuntimeInputs() {
log.Warn().Msg("AI Patrol: No runtime state available for scoped patrol")
p.recordScopedPatrolScopeFailure(start, runID, scope, PatrolScopeResolution{RequestedResourceIDs: append([]string(nil), scope.ResourceIDs...)}, "Patrol runtime state unavailable", "No normal collection state was available for the requested scoped Patrol run.")
return
}
fullState := p.currentPatrolRuntimeState()
requestedScope := scope
resolvedScope, scopeResolution := resolvePatrolScopeState(fullState, scope)
if len(scopeResolution.UnmatchedResourceIDs) > 0 || len(scopeResolution.AmbiguousResourceIDs) > 0 ||
(len(scopeResolution.RequestedResourceIDs) > 0 && len(scopeResolution.ResolvedResourceIDs) == 0) {
p.recordScopedPatrolScopeFailure(start, runID, requestedScope, scopeResolution,
"Patrol scope could not be resolved",
"One or more requested resource identities did not resolve exactly to the current Patrol collection state.")
return
}
scope = resolvedScope
// Filter state based on scope
filteredState := p.filterStateByScopeState(fullState, scope)
effectiveScopeIDs := patrolRuntimeSortedResourceIDs(filteredState)
resourceCounts := patrolRuntimeCountResources(filteredState)
resourceCount := 0
if cfg.AnalyzeNodes {
resourceCount += resourceCounts.nodes
}
if cfg.AnalyzeGuests {
resourceCount += resourceCounts.guests
}
if cfg.AnalyzeDocker {
resourceCount += resourceCounts.docker
}
if cfg.AnalyzeStorage {
resourceCount += resourceCounts.storage
}
if cfg.AnalyzePBS {
resourceCount += resourceCounts.pbs
}
if cfg.AnalyzeHosts {
resourceCount += resourceCounts.hosts
resourceCount += resourceCounts.truenas
}
if cfg.AnalyzeKubernetes {
resourceCount += resourceCounts.kubernetes
}
if cfg.AnalyzePMG {
resourceCount += resourceCounts.pmg
}
// Canonical resource kinds without a legacy snapshot projection are still
// real scoped work. They are admitted only through exact unified-resource
// matching and remain available to the objective seed and governed tools.
resourceCount += resourceCounts.unified
if resourceCount == 0 {
log.Debug().
Strs("requested_ids", scope.ResourceIDs).
Strs("requested_types", scope.ResourceTypes).
Int("effective_scope_count", len(effectiveScopeIDs)).
Msg("AI Patrol: No resources matched scope filter")
p.recordScopedPatrolScopeFailure(start, runID, scope, scopeResolution, "No resources matched requested Patrol scope", "The requested resource identities did not resolve to any enabled resource in the current normal Patrol collection state.")
return
}
log.Debug().
Strs("requested_ids", scope.ResourceIDs).
Strs("requested_types", scope.ResourceTypes).
Strs("effective_scope_ids", patrolLogResourceIDs(effectiveScopeIDs)).
Int("effective_scope_count", len(effectiveScopeIDs)).
Int("resource_count", resourceCount).
Str("reason", string(scope.Reason)).
Msg("AI Patrol: Running scoped analysis")
// Track run statistics
if cfg.AnalyzeNodes {
runStats.nodesChecked = resourceCounts.nodes
}
if cfg.AnalyzeGuests {
runStats.guestsChecked = resourceCounts.guests
}
if cfg.AnalyzeDocker {
runStats.dockerChecked = resourceCounts.docker
}
if cfg.AnalyzeStorage {
runStats.storageChecked = resourceCounts.storage
}
if cfg.AnalyzePBS {
runStats.pbsChecked = resourceCounts.pbs
}
if cfg.AnalyzeHosts {
runStats.hostsChecked = resourceCounts.hosts
runStats.trueNASChecked = resourceCounts.truenas
}
if cfg.AnalyzeKubernetes {
runStats.kubernetesChecked = resourceCounts.kubernetes
}
if cfg.AnalyzePMG {
runStats.pmgChecked = resourceCounts.pmg
}
runStats.resourceCount = resourceCount
// Acquire a breaker probe only after the requested scope has resolved to
// real work. This keeps collection-only failures outside provider health.
aiServiceEnabled := p.retryProviderInitIfNeeded(ctx)
llmAllowed := true
if aiServiceEnabled && breaker != nil {
llmAllowed = breaker.Allow()
}
if !llmAllowed {
log.Warn().Msg("AI Patrol: Circuit breaker is open for scoped patrol (LLM calls blocked)")
}
canRunLLM := aiServiceEnabled && llmAllowed
if !canRunLLM {
reason, cause := p.providerUnavailableReason()
if aiServiceEnabled && !llmAllowed {
reason = "circuit breaker is open"
cause = PatrolFailureCauseCircuitOpen
GetPatrolMetrics().RecordCircuitBlock()
} else {
p.raiseBlockedRunFinding(reason, cause)
}
p.setBlockedReasonWithCause(reason, cause)
log.Info().Str("reason", reason).Str("cause", string(cause)).Msg("AI Patrol: Skipping scoped run - AI unavailable")
return
}
{
p.clearBlockedReason()
if !scope.NoStream {
// Ensure stream state is clean for this run before the first streamed event.
p.resetStreamForRun(runID)
}
// Run agentic AI analysis on filtered state with scope
aiResult, aiErr := p.runAIAnalysisState(ctx, filteredState, &scope, executionID)
if aiResult != nil {
runStats.aiAnalysis = aiResult
runStats.rejectedFindings = aiResult.RejectedFindings
runStats.triageFlags = aiResult.TriageFlags
runStats.triageSkippedLLM = aiResult.TriageSkippedLLM
// Keep accepted finding evidence attached to this run even if the
// provider fails after the corresponding tool call completed.
for _, f := range aiResult.Findings {
if f.Severity == FindingSeverityWarning || f.Severity == FindingSeverityCritical {
runStats.findingIDs = append(runStats.findingIDs, f.ID)
stored := p.findings.Get(f.ID)
if stored != nil && stored.TimesRaised <= 1 {
runStats.newFindings++
} else {
runStats.existingFindings++
}
}
}
}
if aiErr != nil {
log.Warn().Err(aiErr).Msg("AI Patrol (scoped): LLM analysis failed")
runStats.errors++
runStats.lastAIError = aiErr
failure := patrolRuntimeFailureFromError(aiErr)
runStats.errorSummary = failure.Summary
runStats.errorDetail = failure.Detail
p.blockOnExhaustedBudget(aiErr, failure)
errorFinding := newPatrolRuntimeFailureFinding(failure, time.Now())
if p.recordFinding(errorFinding) {
runStats.newFindings++
} else {
runStats.existingFindings++
}
runStats.findingIDs = append(runStats.findingIDs, errorFinding.ID)
} else if aiResult != nil {
if missing := patrolMissingAssessmentIDs(aiResult); len(missing) > 0 {
runStats.errors++
runStats.errorSummary = "Patrol finding assessment incomplete"
runStats.errorDetail = fmt.Sprintf("No explicit verdict was accepted for active finding IDs: %s", strings.Join(missing, ", "))
}
if !aiResult.TriageSkippedLLM {
runStats.runtimeResolved = p.resolvePatrolRuntimeFailureFinding("scoped_patrol_success")
}
}
}
duration := time.Since(start)
completedAt := time.Now()
// Build a run-owned findings summary. Supporting resources included in the
// effective scope provide diagnostic context, but an unrelated finding on
// one of those resources must not make this targeted run appear unhealthy.
runFindingIDs := patrolRunFindingIDs(runStats.findingIDs, assessmentsForRun(runStats.aiAnalysis))
summary := patrolFindingSummaryForRun(p.findings.GetActive(FindingSeverityWarning), runFindingIDs)
var findingsSummaryStr string
var status string
totalActive := summary.Critical + summary.Warning
if totalActive == 0 {
findingsSummaryStr = "All healthy"
status = "healthy"
} else {
parts := []string{}
if summary.Critical > 0 {
parts = append(parts, fmt.Sprintf("%d critical", summary.Critical))
}
if summary.Warning > 0 {
parts = append(parts, fmt.Sprintf("%d warning", summary.Warning))
}
findingsSummaryStr = fmt.Sprintf("%s", joinParts(parts))
if summary.Critical > 0 {
status = "critical"
} else {
status = "issues_found"
}
}
if runStats.errors > 0 {
status = "error"
if findingsSummaryStr == "All healthy" {
findingsSummaryStr = fmt.Sprintf("Analysis incomplete (%d errors)", runStats.errors)
}
}
resolvedFindings := 0
if runStats.aiAnalysis != nil {
resolvedFindings = len(runStats.aiAnalysis.ResolvedIDs)
if runStats.runtimeResolved {
resolvedFindings++
}
}
runRecord := PatrolRunRecord{
ID: runID,
StartedAt: start,
CompletedAt: completedAt,
Duration: duration,
DurationMs: duration.Milliseconds(),
Type: "scoped",
TriggerReason: string(scope.Reason),
ScopeResourceIDs: scopeResolution.RequestedResourceIDs,
EffectiveScopeResourceIDs: effectiveScopeIDs,
ScopeResourceTypes: scope.ResourceTypes,
ScopeContext: scope.Context,
ObjectiveContext: clonePatrolObjectiveContext(scope.ObjectiveContext),
AlertIdentifier: scope.AlertIdentifier,
FindingID: scope.FindingID,
ResourcesChecked: runStats.resourceCount,
NodesChecked: runStats.nodesChecked,
GuestsChecked: runStats.guestsChecked,
DockerChecked: runStats.dockerChecked,
StorageChecked: runStats.storageChecked,
HostsChecked: runStats.hostsChecked,
TrueNASChecked: runStats.trueNASChecked,
PBSChecked: runStats.pbsChecked,
PMGChecked: runStats.pmgChecked,
KubernetesChecked: runStats.kubernetesChecked,
NewFindings: runStats.newFindings,
ExistingFindings: runStats.existingFindings,
RejectedFindings: runStats.rejectedFindings,
ResolvedFindings: resolvedFindings,
FindingsSummary: findingsSummaryStr,
FindingIDs: runFindingIDs,
ErrorCount: runStats.errors,
Status: status,
ErrorSummary: runStats.errorSummary,
ErrorDetail: runStats.errorDetail,
}
if runStats.aiAnalysis != nil {
runRecord.AIAnalysis = patrolRunAIAnalysisForRecord(runStats.aiAnalysis, patrolRunAnalysisRecordContext{
ResourcesChecked: runStats.resourceCount,
NewFindings: runStats.newFindings,
ExistingFindings: runStats.existingFindings,
ResolvedFindings: resolvedFindings,
UncertainFindings: patrolUncertainAssessmentCount(runStats.aiAnalysis.Assessments),
ErrorCount: runStats.errors,
})
runRecord.FindingAssessments = append([]PatrolFindingAssessment(nil), runStats.aiAnalysis.Assessments...)
runRecord.InputTokens = runStats.aiAnalysis.InputTokens
runRecord.OutputTokens = runStats.aiAnalysis.OutputTokens
runRecord.CacheCreationInputTokens = runStats.aiAnalysis.CacheCreationInputTokens
runRecord.CacheReadInputTokens = runStats.aiAnalysis.CacheReadInputTokens
runRecord.TriageFlags = runStats.triageFlags
runRecord.TriageSkippedLLM = runStats.triageSkippedLLM
toolCalls := runStats.aiAnalysis.ToolCalls
if len(toolCalls) > MaxToolCallsPerRun {
toolCalls = toolCalls[:MaxToolCallsPerRun]
}
runRecord.ToolCalls = toolCalls
runRecord.ToolCallCount = len(runStats.aiAnalysis.ToolCalls)
}
p.mu.Lock()
p.lastActivity = completedAt
p.lastDuration = duration
p.resourcesChecked = runStats.resourceCount
p.errorCount = runStats.errors
p.mu.Unlock()
recordPatrolCircuitResult(breaker, canRunLLM, runStats.errors, runStats.lastAIError)
p.runHistoryStore.Add(runRecord)
log.Info().
Strs("requested_ids", scopeResolution.RequestedResourceIDs).
Strs("requested_types", scope.ResourceTypes).
Strs("effective_scope_ids", patrolLogResourceIDs(effectiveScopeIDs)).
Int("effective_scope_count", len(effectiveScopeIDs)).
Dur("duration", duration).
Int("resources", resourceCount).
Str("reason", string(scope.Reason)).
Msg("AI Patrol: Scoped patrol complete")
p.dispatchPatrolInvestigations(runStats.aiAnalysis)
}
func recordPatrolCircuitResult(breaker *circuit.Breaker, attempted bool, errorCount int, runErr error) {
if breaker == nil || !attempted {
return
}
if errorCount == 0 {
breaker.RecordSuccess()
return
}
if runErr == nil {
runErr = fmt.Errorf("patrol completed with %d errors", errorCount)
}
if errors.Is(runErr, ErrCostBudgetExceeded) {
// Pulse declined to spend; the provider was never called, so there
// is no provider failure to count (#1789).
return
}
breaker.RecordFailureWithCategory(runErr, circuit.CategorizeError(runErr))
}
func (p *PatrolService) recordScopedPatrolScopeFailure(start time.Time, runID string, scope PatrolScope, resolution PatrolScopeResolution, summary, detail string) {
completedAt := time.Now()
record := PatrolRunRecord{
ID: runID,
StartedAt: start,
CompletedAt: completedAt,
Duration: completedAt.Sub(start),
DurationMs: completedAt.Sub(start).Milliseconds(),
Type: "scoped",
TriggerReason: string(scope.Reason),
ScopeResourceIDs: append([]string(nil), resolution.RequestedResourceIDs...),
EffectiveScopeResourceIDs: append([]string(nil), resolution.EffectiveResourceIDs...),
ScopeResourceTypes: append([]string(nil), scope.ResourceTypes...),
ScopeContext: scope.Context,
ObjectiveContext: clonePatrolObjectiveContext(scope.ObjectiveContext),
AlertIdentifier: scope.AlertIdentifier,
FindingID: scope.FindingID,
FindingIDs: []string{},
FindingsSummary: "Analysis incomplete (scope resolution failed)",
ErrorCount: 1,
Status: "error",
ErrorSummary: summary,
ErrorDetail: detail,
}
p.mu.Lock()
p.lastActivity = completedAt
p.lastDuration = record.Duration
p.resourcesChecked = 0
p.errorCount = 1
p.mu.Unlock()
p.runHistoryStore.Add(record)
}
func patrolLogResourceIDs(ids []string) []string {
if len(ids) <= scopedPatrolLogIDLimit {
return ids
}
trimmed := append([]string(nil), ids[:scopedPatrolLogIDLimit]...)
trimmed = append(trimmed, fmt.Sprintf("... +%d more", len(ids)-scopedPatrolLogIDLimit))
return trimmed
}
type patrolScopeMatcher struct {
resourceIDSet map[string]bool
typeSet map[string]bool
hasIDs bool
hasTypes bool
identityOnly bool
}
func newPatrolScopeMatcher(scope PatrolScope) patrolScopeMatcher {
resourceIDSet := make(map[string]bool)
for _, id := range scope.ResourceIDs {
trimmed := strings.TrimSpace(id)
if trimmed == "" {
continue
}
resourceIDSet[trimmed] = true
}
typeSet := make(map[string]bool)
addScopeType := func(t string) {
trimmed := strings.TrimSpace(strings.ToLower(t))
if trimmed == "" {
return
}
switch trimmed {
case "docker-host", "app-container":
typeSet["docker-host"] = true
typeSet["app-container"] = true
case "k8s-cluster":
typeSet["k8s-cluster"] = true
case "system-container", "vm", "node", "storage", "agent", "pbs", "pmg", "physical_disk":
typeSet[trimmed] = true
default:
typeSet[trimmed] = true
}
}
for _, t := range scope.ResourceTypes {
addScopeType(t)
}
return patrolScopeMatcher{
resourceIDSet: resourceIDSet,
typeSet: typeSet,
hasIDs: len(resourceIDSet) > 0,
hasTypes: len(typeSet) > 0,
identityOnly: scope.resolvedIdentityOnly,
}
}
func (m patrolScopeMatcher) matchesType(candidates ...string) bool {
if !m.hasTypes {
return true
}
for _, candidate := range candidates {
if candidate == "" {
continue
}
if m.typeSet[strings.ToLower(candidate)] {
return true
}
}
return false
}
func (m patrolScopeMatcher) matchesResource(ids []string, aliases ...string) bool {
if !m.hasIDs {
return true
}
for _, candidate := range ids {
if candidate == "" {
continue
}
if m.resourceIDSet[candidate] {
return true
}
}
if m.identityOnly {
return false
}
for _, candidate := range aliases {
if candidate != "" && m.resourceIDSet[candidate] {
return true
}
}
return false
}
type patrolScopedFilterState struct {
filtered patrolRuntimeState
includedResourceIDs map[string]bool
includedGuestVMIDs map[int]bool
}
func newPatrolScopedFilterState(snap patrolRuntimeState) patrolScopedFilterState {
return patrolScopedFilterState{
filtered: patrolRuntimeState{
readState: snap.readState,
unifiedResourceProvider: snap.unifiedResourceProvider,
thresholdProvider: snap.thresholdProvider,
},
includedResourceIDs: make(map[string]bool),
includedGuestVMIDs: make(map[int]bool),
}
}
func (s *patrolScopedFilterState) includeResourceID(ids ...string) {
for _, id := range ids {
if strings.TrimSpace(id) == "" {
continue
}
s.includedResourceIDs[id] = true
}
}
func (s *patrolScopedFilterState) includeGuestVMID(vmid int) {
if vmid > 0 {
s.includedGuestVMIDs[vmid] = true
}
}
func patrolDockerScopeName(d models.DockerHost) string {
hostName := d.CustomDisplayName
if hostName == "" {
hostName = d.DisplayName
}
if hostName == "" {
hostName = d.Hostname
}
return hostName
}
func scopePatrolDockerHost(d models.DockerHost, matcher patrolScopeMatcher) (models.DockerHost, []string, bool) {
if !matcher.matchesType("docker-host", "app-container") {
return models.DockerHost{}, nil, false
}
hostMatches := matcher.matchesResource([]string{d.ID}, patrolDockerScopeName(d), d.Hostname, d.DisplayName, d.CustomDisplayName)
if !matcher.hasIDs {
included := make([]string, 0, len(d.Containers)+1)
if matcher.typeSet["docker-host"] || !matcher.hasTypes {
included = append(included, d.ID)
}
if matcher.typeSet["app-container"] || !matcher.hasTypes {
for _, c := range d.Containers {
included = append(included, c.ID)
}
}
return d, included, true
}
matchedContainers := make([]models.DockerContainer, 0)
for _, c := range d.Containers {
if matcher.matchesResource([]string{c.ID}, c.Name) {
matchedContainers = append(matchedContainers, c)
}
}
if hostMatches {
included := make([]string, 0, len(d.Containers)+1)
if matcher.typeSet["docker-host"] || !matcher.hasTypes {
included = append(included, d.ID)
}
if matcher.typeSet["app-container"] || !matcher.hasTypes {
for _, c := range d.Containers {
included = append(included, c.ID)
}
}
return d, included, true
}
if len(matchedContainers) > 0 {
hostCopy := d
hostCopy.Containers = matchedContainers
// A Docker host is the transport parent for its containers, so the
// filtered snapshot must retain the host record. It must not, however,
// retain the host's complete child inventory. Re-ingesting that inventory
// into the scoped unified-resource provider would turn a request for one
// container into every image, volume, network, and Swarm object on the
// host. The matched container already carries its own image, mounts, and
// network attachments as bounded dependency evidence.
hostCopy.Images = nil
hostCopy.Volumes = nil
hostCopy.Networks = nil
hostCopy.Services = nil
hostCopy.Tasks = nil
hostCopy.Nodes = nil
hostCopy.Secrets = nil
hostCopy.Configs = nil
hostCopy.StorageUsage = nil
hostCopy.Swarm = nil
included := make([]string, 0, len(matchedContainers))
for _, c := range matchedContainers {
included = append(included, c.ID)
}
return hostCopy, included, true
}
return models.DockerHost{}, nil, false
}
func patrolUpdateSafetyDockerHosts(s patrolRuntimeState) []models.DockerHost {
rs := s.readState
if rs == nil {
derived := s.withDerivedProviders()
rs = derived.readState
}
if rs == nil {
return nil
}
hostViews := rs.DockerHosts()
hosts := make([]models.DockerHost, 0, len(hostViews))
for _, hostView := range hostViews {
if hostView == nil {
continue
}
hostID := strings.TrimSpace(hostView.HostSourceID())
if hostID == "" {
hostID = strings.TrimSpace(hostView.ID())
}
name := strings.TrimSpace(hostView.Name())
displayName := strings.TrimSpace(hostView.DisplayName())
if displayName == "" {
displayName = name
}
hostname := strings.TrimSpace(hostView.Hostname())
if hostname == "" {
hostname = name
}
hosts = append(hosts, models.DockerHost{
ID: hostID,
AgentID: strings.TrimSpace(hostView.AgentID()),
Hostname: hostname,
DisplayName: displayName,
CustomDisplayName: strings.TrimSpace(hostView.CustomDisplayName()),
Status: string(hostView.Status()),
Containers: hostView.Containers(),
})
}
return hosts
}
func collectPatrolScopedDockerHosts(hosts []models.DockerHost, matcher patrolScopeMatcher) ([]models.DockerHost, []string) {
filtered := make([]models.DockerHost, 0, len(hosts))
ids := make([]string, 0, len(hosts))
for _, host := range hosts {
scopedHost, includeIDs, ok := scopePatrolDockerHost(host, matcher)
if !ok {
continue
}
filtered = append(filtered, scopedHost)
ids = append(ids, includeIDs...)
}
return filtered, ids
}
func scopePatrolPBSInstance(pbs models.PBSInstance, matcher patrolScopeMatcher) bool {
if !matcher.matchesType("pbs") {
return false
}
pbsName := pbs.Name
if pbsName == "" {
pbsName = pbs.Host
}
pbsMatches := matcher.matchesResource([]string{pbs.ID}, pbs.Name, pbsName, pbs.Host)
if !matcher.hasIDs {
return true
}
if !pbsMatches {
for _, ds := range pbs.Datastores {
if matcher.matchesResource([]string{pbs.ID + ":" + ds.Name}, ds.Name) {
pbsMatches = true
break
}
}
}
if !pbsMatches {
for _, job := range pbs.BackupJobs {
if matcher.matchesResource([]string{pbs.ID + ":job:" + job.ID}, job.ID) {
pbsMatches = true
break
}
}
}
if !pbsMatches {
for _, job := range pbs.VerifyJobs {
if matcher.matchesResource([]string{pbs.ID + ":verify:" + job.ID}, job.ID) {
pbsMatches = true
break
}
}
}
return pbsMatches
}
func scopePatrolNode(n models.Node, matcher patrolScopeMatcher) bool {
return matcher.matchesType("node") && matcher.matchesResource([]string{n.ID}, n.Name)
}
func scopePatrolVM(vm models.VM, matcher patrolScopeMatcher) bool {
return matcher.matchesType("vm") && matcher.matchesResource([]string{vm.ID}, vm.Name)
}
func scopePatrolContainer(ct models.Container, matcher patrolScopeMatcher) bool {
return matcher.matchesType("system-container") && matcher.matchesResource([]string{ct.ID}, ct.Name)
}
func scopePatrolStorage(storage models.Storage, matcher patrolScopeMatcher) bool {
return matcher.matchesType("storage") && matcher.matchesResource([]string{storage.ID}, storage.Name)
}
func scopePatrolPhysicalDisk(disk models.PhysicalDisk, matcher patrolScopeMatcher) bool {
return matcher.matchesType("physical_disk") && matcher.matchesResource([]string{disk.ID, disk.DevPath}, disk.Model)
}
func scopePatrolPMGInstance(pmg models.PMGInstance, matcher patrolScopeMatcher) bool {
return matcher.matchesType("pmg") && matcher.matchesResource([]string{pmg.ID}, pmg.Name, pmg.Host)
}
func scopePatrolHost(h models.Host, matcher patrolScopeMatcher) bool {
return matcher.matchesType("agent") && matcher.matchesResource([]string{h.ID}, h.DisplayName, h.Hostname)
}
func scopePatrolKubernetesCluster(k models.KubernetesCluster, matcher patrolScopeMatcher) bool {
return matcher.matchesType("k8s-cluster") && matcher.matchesResource([]string{k.ID}, patrolKubernetesScopeName(k))
}
func collectPatrolScopedNodes(nodes []models.Node, matcher patrolScopeMatcher) ([]models.Node, []string) {
filtered := make([]models.Node, 0, len(nodes))
ids := make([]string, 0, len(nodes))
for _, n := range nodes {
if !scopePatrolNode(n, matcher) {
continue
}
filtered = append(filtered, n)
ids = append(ids, n.ID)
}
return filtered, ids
}
func collectPatrolScopedVMs(vms []models.VM, matcher patrolScopeMatcher) ([]models.VM, []string, []int) {
filtered := make([]models.VM, 0, len(vms))
ids := make([]string, 0, len(vms))
vmids := make([]int, 0, len(vms))
for _, vm := range vms {
if !scopePatrolVM(vm, matcher) {
continue
}
filtered = append(filtered, vm)
ids = append(ids, vm.ID)
vmids = append(vmids, vm.VMID)
}
return filtered, ids, vmids
}
func collectPatrolScopedContainers(containers []models.Container, matcher patrolScopeMatcher) ([]models.Container, []string, []int) {
filtered := make([]models.Container, 0, len(containers))
ids := make([]string, 0, len(containers))
vmids := make([]int, 0, len(containers))
for _, ct := range containers {
if !scopePatrolContainer(ct, matcher) {
continue
}
filtered = append(filtered, ct)
ids = append(ids, ct.ID)
vmids = append(vmids, ct.VMID)
}
return filtered, ids, vmids
}
func collectPatrolScopedStorage(storage []models.Storage, matcher patrolScopeMatcher) ([]models.Storage, []string) {
filtered := make([]models.Storage, 0, len(storage))
ids := make([]string, 0, len(storage))
for _, s := range storage {
if !scopePatrolStorage(s, matcher) {
continue
}
filtered = append(filtered, s)
ids = append(ids, s.ID)
}
return filtered, ids
}
func collectPatrolScopedPhysicalDisks(disks []models.PhysicalDisk, matcher patrolScopeMatcher) ([]models.PhysicalDisk, []string) {
filtered := make([]models.PhysicalDisk, 0, len(disks))
ids := make([]string, 0, len(disks)*2)
for _, disk := range disks {
if !scopePatrolPhysicalDisk(disk, matcher) {
continue
}
filtered = append(filtered, disk)
ids = append(ids, disk.ID, disk.DevPath)
}
return filtered, ids
}
func collectPatrolScopedPBSInstances(instances []models.PBSInstance, matcher patrolScopeMatcher) ([]models.PBSInstance, []string) {
filtered := make([]models.PBSInstance, 0, len(instances))
ids := make([]string, 0, len(instances))
for _, pbs := range instances {
if !scopePatrolPBSInstance(pbs, matcher) {
continue
}
filtered = append(filtered, pbs)
ids = append(ids, pbs.ID)
}
return filtered, ids
}
func collectPatrolScopedPMGInstances(instances []models.PMGInstance, matcher patrolScopeMatcher) ([]models.PMGInstance, []string) {
filtered := make([]models.PMGInstance, 0, len(instances))
ids := make([]string, 0, len(instances))
for _, pmg := range instances {
if !scopePatrolPMGInstance(pmg, matcher) {
continue
}
filtered = append(filtered, pmg)
ids = append(ids, pmg.ID)
}
return filtered, ids
}
func collectPatrolScopedHosts(hosts []models.Host, matcher patrolScopeMatcher) ([]models.Host, []string) {
filtered := make([]models.Host, 0, len(hosts))
ids := make([]string, 0, len(hosts))
for _, h := range hosts {
if !scopePatrolHost(h, matcher) {
continue
}
filtered = append(filtered, h)
ids = append(ids, h.ID)
}
return filtered, ids
}
func collectPatrolScopedKubernetesClusters(clusters []models.KubernetesCluster, matcher patrolScopeMatcher) ([]models.KubernetesCluster, []string) {
filtered := make([]models.KubernetesCluster, 0, len(clusters))
ids := make([]string, 0, len(clusters))
for _, k := range clusters {
if !scopePatrolKubernetesCluster(k, matcher) {
continue
}
filtered = append(filtered, k)
ids = append(ids, k.ID)
}
return filtered, ids
}
func patrolUnifiedResourceIdentity(resource unifiedresources.Resource) ([]string, []string) {
ids := []string{resource.ID}
aliases := []string{resource.Name}
if resource.Canonical != nil {
ids = append(ids, resource.Canonical.PrimaryID)
ids = append(ids, resource.Canonical.SupersededIDs...)
aliases = append(aliases, resource.Canonical.DisplayName, resource.Canonical.Hostname)
aliases = append(aliases, resource.Canonical.Aliases...)
}
ids = append(ids, resource.SupersededCanonicalIDs...)
return ids, aliases
}
func collectPatrolScopedUnifiedResources(provider UnifiedResourceProvider, matcher patrolScopeMatcher) ([]unifiedresources.Resource, []string) {
if provider == nil {
return nil, nil
}
filtered := make([]unifiedresources.Resource, 0)
ids := make([]string, 0)
for _, resource := range provider.GetAll() {
resourceIDs, aliases := patrolUnifiedResourceIdentity(resource)
if !matcher.matchesType(string(resource.Type)) || !matcher.matchesResource(resourceIDs, aliases...) {
continue
}
filtered = append(filtered, resource)
ids = append(ids, resourceIDs...)
}
return filtered, ids
}
func patrolKubernetesScopeName(k models.KubernetesCluster) string {
clusterName := k.CustomDisplayName
if clusterName == "" {
clusterName = k.DisplayName
}
if clusterName == "" {
clusterName = k.Name
}
return clusterName
}
func collectPatrolScopedActiveAlerts(state patrolRuntimeState, alerts []models.Alert, includedResourceIDs map[string]bool) []models.Alert {
filtered := make([]models.Alert, 0, len(alerts))
for _, alert := range alerts {
if patrolAlertBelongsToScope(state, alert.ResourceID, includedResourceIDs) {
filtered = append(filtered, alert)
}
}
return filtered
}
func collectPatrolScopedResolvedAlerts(state patrolRuntimeState, alerts []models.ResolvedAlert, includedResourceIDs map[string]bool) []models.ResolvedAlert {
filtered := make([]models.ResolvedAlert, 0, len(alerts))
for _, resolved := range alerts {
if patrolAlertBelongsToScope(state, resolved.ResourceID, includedResourceIDs) {
filtered = append(filtered, resolved)
}
}
return filtered
}
func collectPatrolScopedConnectionHealth(connectionHealth map[string]bool, includedResourceIDs map[string]bool) map[string]bool {
filtered := make(map[string]bool, len(connectionHealth))
for resourceID, healthy := range connectionHealth {
if includedResourceIDs[resourceID] {
filtered[resourceID] = healthy
}
}
return filtered
}
func collectPatrolScopedBackupTasks(tasks []models.BackupTask, includedGuestVMIDs map[int]bool) []models.BackupTask {
filtered := make([]models.BackupTask, 0, len(tasks))
for _, backupTask := range tasks {
if includedGuestVMIDs[backupTask.VMID] {
filtered = append(filtered, backupTask)
}
}
return filtered
}
func collectPatrolScopedStorageBackups(backups []models.StorageBackup, includedGuestVMIDs map[int]bool) []models.StorageBackup {
filtered := make([]models.StorageBackup, 0, len(backups))
for _, storageBackup := range backups {
if includedGuestVMIDs[storageBackup.VMID] {
filtered = append(filtered, storageBackup)
}
}
return filtered
}
func collectPatrolScopedGuestSnapshots(snapshots []models.GuestSnapshot, includedGuestVMIDs map[int]bool) []models.GuestSnapshot {
filtered := make([]models.GuestSnapshot, 0, len(snapshots))
for _, guestSnapshot := range snapshots {
if includedGuestVMIDs[guestSnapshot.VMID] {
filtered = append(filtered, guestSnapshot)
}
}
return filtered
}
func collectPatrolScopedPBSBackups(backups []models.PBSBackup, includedGuestVMIDs map[int]bool) []models.PBSBackup {
filtered := make([]models.PBSBackup, 0, len(backups))
for _, backup := range backups {
vmid, err := strconv.Atoi(backup.VMID)
if err == nil && includedGuestVMIDs[vmid] {
filtered = append(filtered, backup)
}
}
return filtered
}
func copyScopedPatrolMetadata(dst *patrolRuntimeState, snap patrolRuntimeState, includedResourceIDs map[string]bool, includedGuestVMIDs map[int]bool) {
if len(snap.ActiveAlerts) > 0 {
dst.ActiveAlerts = collectPatrolScopedActiveAlerts(snap, snap.ActiveAlerts, includedResourceIDs)
}
if len(snap.RecentlyResolved) > 0 {
dst.RecentlyResolved = collectPatrolScopedResolvedAlerts(snap, snap.RecentlyResolved, includedResourceIDs)
}
if len(snap.ConnectionHealth) > 0 {
dst.ConnectionHealth = collectPatrolScopedConnectionHealth(snap.ConnectionHealth, includedResourceIDs)
}
if len(includedGuestVMIDs) == 0 {
return
}
dst.PVEBackups.BackupTasks = collectPatrolScopedBackupTasks(snap.PVEBackups.BackupTasks, includedGuestVMIDs)
dst.PVEBackups.StorageBackups = collectPatrolScopedStorageBackups(snap.PVEBackups.StorageBackups, includedGuestVMIDs)
dst.PVEBackups.GuestSnapshots = collectPatrolScopedGuestSnapshots(snap.PVEBackups.GuestSnapshots, includedGuestVMIDs)
dst.PBSBackups = collectPatrolScopedPBSBackups(snap.PBSBackups, includedGuestVMIDs)
}
func filterPatrolStateByScopeState(snap patrolRuntimeState, scope PatrolScope) patrolRuntimeState {
if len(snap.ActiveAlerts)+len(snap.RecentlyResolved) > 0 {
snap.alertResourceResolver = patrolAlertReferenceResolver(snap)
}
matcher := newPatrolScopeMatcher(scope)
filterState := newPatrolScopedFilterState(snap)
filteredNodes, nodeIDs := collectPatrolScopedNodes(snap.Nodes, matcher)
filterState.filtered.Nodes = filteredNodes
filterState.includeResourceID(nodeIDs...)
filteredVMs, vmIDs, guestVMIDs := collectPatrolScopedVMs(snap.VMs, matcher)
filterState.filtered.VMs = filteredVMs
filterState.includeResourceID(vmIDs...)
for _, vmid := range guestVMIDs {
filterState.includeGuestVMID(vmid)
}
filteredContainers, containerIDs, containerVMIDs := collectPatrolScopedContainers(snap.Containers, matcher)
filterState.filtered.Containers = filteredContainers
filterState.includeResourceID(containerIDs...)
for _, vmid := range containerVMIDs {
filterState.includeGuestVMID(vmid)
}
filteredDockerHosts, dockerIDs := collectPatrolScopedDockerHosts(snap.DockerHosts, matcher)
filterState.filtered.DockerHosts = filteredDockerHosts
filterState.includeResourceID(dockerIDs...)
filteredStorage, storageIDs := collectPatrolScopedStorage(snap.Storage, matcher)
filterState.filtered.Storage = filteredStorage
filterState.includeResourceID(storageIDs...)
filteredDisks, diskIDs := collectPatrolScopedPhysicalDisks(snap.PhysicalDisks, matcher)
filterState.filtered.PhysicalDisks = filteredDisks
filterState.includeResourceID(diskIDs...)
filteredPBS, pbsIDs := collectPatrolScopedPBSInstances(snap.PBSInstances, matcher)
filterState.filtered.PBSInstances = filteredPBS
filterState.includeResourceID(pbsIDs...)
filteredPMG, pmgIDs := collectPatrolScopedPMGInstances(snap.PMGInstances, matcher)
filterState.filtered.PMGInstances = filteredPMG
filterState.includeResourceID(pmgIDs...)
filteredHosts, hostIDs := collectPatrolScopedHosts(snap.Hosts, matcher)
filterState.filtered.Hosts = filteredHosts
filterState.includeResourceID(hostIDs...)
filteredK8sClusters, k8sClusterIDs := collectPatrolScopedKubernetesClusters(snap.KubernetesClusters, matcher)
filterState.filtered.KubernetesClusters = filteredK8sClusters
filterState.includeResourceID(k8sClusterIDs...)
// Preserve exact canonical resources that have no StateSnapshot projection
// (notably availability network endpoints). Rebuilding the provider only
// from snapshot fields made valid objective scopes disappear before the
// model could propose a local observer. The scoped provider is still built
// from the exact matcher result, so unrelated resources cannot enter model
// context.
filteredUnified, unifiedIDs := collectPatrolScopedUnifiedResources(snap.unifiedResourceProvider, matcher)
filterState.includeResourceID(unifiedIDs...)
if len(filteredUnified) > 0 {
filterState.filtered.alertResourceResolver = patrolAlertReferenceResolver(snap)
}
copyScopedPatrolMetadata(&filterState.filtered, snap, filterState.includedResourceIDs, filterState.includedGuestVMIDs)
return filterState.filtered.withDerivedProvidersAndResources(filteredUnified)
}
func (p *PatrolService) filterStateByScopeState(snap patrolRuntimeState, scope PatrolScope) patrolRuntimeState {
return filterPatrolStateByScopeState(snap, scope)
}
// GetStatus returns the current patrol status
func (p *PatrolService) GetStatus() PatrolStatus {
p.mu.RLock()
defer p.mu.RUnlock()
interval := p.config.GetInterval()
intervalMs := int64(interval / time.Millisecond)
// "Running" means an analysis is currently in progress, not just the service loop
analysisInProgress := p.runInProgress
status := PatrolStatus{
RuntimeState: PatrolRuntimeStateActive,
Running: analysisInProgress,
CurrentRunID: p.currentRunID,
Enabled: p.config.Enabled,
LastDuration: p.lastDuration,
ResourcesChecked: p.resourcesChecked,
FindingsCount: len(p.findings.GetActive(FindingSeverityInfo)),
ErrorCount: p.errorCount,
IntervalMs: intervalMs,
BlockedReason: p.lastBlockedReason,
BlockedCause: p.lastBlockedCause,
}
if analysisInProgress && !p.runStartedAt.IsZero() {
startedAt := p.runStartedAt
status.CurrentRunStartedAt = &startedAt
}
if p.triggerManager != nil {
triggerStatus := p.triggerManager.GetStatus()
status.TriggerStatus = &triggerStatus
}
switch {
case analysisInProgress:
status.RuntimeState = PatrolRuntimeStateRunning
case !p.config.Enabled:
status.RuntimeState = PatrolRuntimeStateDisabled
case strings.TrimSpace(status.BlockedReason) != "":
status.RuntimeState = PatrolRuntimeStateBlocked
default:
status.RuntimeState = PatrolRuntimeStateActive
}
if !p.lastFullPatrol.IsZero() {
status.LastPatrolAt = &p.lastFullPatrol
}
if !p.lastActivity.IsZero() {
status.LastActivityAt = &p.lastActivity
}
if strings.TrimSpace(status.BlockedReason) != "" && !p.lastBlockedAt.IsZero() {
status.BlockedAt = &p.lastBlockedAt
}
// Use the tracked next scheduled time (accounts for ticker resets on interval changes)
if p.config.Enabled && interval > 0 && !p.nextScheduledAt.IsZero() {
next := p.nextScheduledAt
status.NextPatrolAt = &next
}
summary := p.findings.GetSummary()
status.Healthy = summary.IsHealthy()
if status.RuntimeState == PatrolRuntimeStateBlocked {
status.Healthy = false
}
return status
}
// SubscribeToStream returns a channel that will receive streaming patrol events
func (p *PatrolService) SubscribeToStream() chan PatrolStreamEvent {
return p.SubscribeToStreamFrom(0)
}
// SubscribeToStreamFrom subscribes a client to patrol streaming events and optionally replays
// events with Seq > lastSeq (best-effort). This allows SSE clients to resume after disconnects
// using the Last-Event-ID header.
func (p *PatrolService) SubscribeToStreamFrom(lastSeq int64) chan PatrolStreamEvent {
ch := make(chan PatrolStreamEvent, 100) // Buffered to prevent blocking
sub := &streamSubscriber{ch: ch}
replayedCount := 0
snapshotReasons := make([]string, 0, 2)
snapshotReasonSeen := make(map[string]struct{}, 2)
p.streamMu.Lock()
p.streamSubscribers[ch] = sub
trySendSnapshot := func(reason string) bool {
if _, seen := snapshotReasonSeen[reason]; seen {
return true
}
snap := p.makeSnapshotLocked(reason)
select {
case ch <- snap:
snapshotReasonSeen[reason] = struct{}{}
snapshotReasons = append(snapshotReasons, reason)
return true
default:
return false
}
}
bufferStart, bufferEnd := p.streamBufferWindowLocked()
// If the client is behind the buffered window, proactively emit a snapshot that
// advertises truncation. (We may still replay what we have.)
if lastSeq > 0 && bufferStart > 0 && lastSeq < bufferStart && p.streamPhase != "idle" {
trySendSnapshot("buffer_rotated")
}
// Best-effort replay / snapshot:
// - If client provides lastSeq, replay newer buffered events (Seq > lastSeq).
// - If lastSeq is stale/ahead (e.g. from a different run), send a snapshot so UI can resync.
// - If no lastSeq, send a snapshot (late-joiner).
if lastSeq > 0 && len(p.streamEvents) > 0 {
events := p.streamEventsSinceLocked(lastSeq)
replayLoop:
for _, ev := range events {
select {
case ch <- ev:
replayedCount++
default:
// If subscriber can't catch up, stop replaying and let it receive live events.
break replayLoop
}
}
}
if replayedCount == 0 && len(snapshotReasons) == 0 && lastSeq > 0 && p.streamPhase != "idle" {
// lastSeq is likely stale (ahead of this run) or we're missing buffered events.
// Provide a snapshot to allow the UI to resync.
reason := "stale_last_event_id"
if bufferEnd > 0 && lastSeq > bufferEnd {
reason = "stale_last_event_id"
} else if bufferStart > 0 && lastSeq < bufferStart {
reason = "buffer_rotated"
}
trySendSnapshot(reason)
}
if lastSeq == 0 && p.streamPhase != "idle" {
trySendSnapshot("late_joiner")
}
p.streamMu.Unlock()
metrics := GetPatrolMetrics()
if replayedCount > 0 {
metrics.RecordStreamReplay(replayedCount)
log.Debug().Int64("last_seq", lastSeq).Int("replayed_events", replayedCount).Msg("patrol stream replayed buffered events")
}
for _, reason := range snapshotReasons {
metrics.RecordStreamSnapshot(reason)
log.Debug().Int64("last_seq", lastSeq).Str("resync_reason", reason).Msg("patrol stream sent synthetic snapshot")
}
if lastSeq > 0 && replayedCount == 0 && len(snapshotReasons) == 0 {
metrics.RecordStreamMiss()
log.Debug().Int64("last_seq", lastSeq).Msg("patrol stream resume had no replay or snapshot")
}
return ch
}
// UnsubscribeFromStream removes a subscriber
func (p *PatrolService) UnsubscribeFromStream(ch chan PatrolStreamEvent) {
p.streamMu.Lock()
sub, exists := p.streamSubscribers[ch]
delete(p.streamSubscribers, ch)
p.streamMu.Unlock()
// Use atomic CAS to ensure exactly one goroutine closes the channel,
// even if broadcast and unsubscribe race.
if exists && sub.closed.CompareAndSwap(false, true) {
close(ch)
}
}
// broadcast sends an event to all subscribers
// Subscribers with full channels are automatically removed to prevent memory leaks
func (p *PatrolService) broadcast(event PatrolStreamEvent) {
p.streamMu.Lock()
defer p.streamMu.Unlock()
// Track a couple pieces of best-effort state for snapshots/resync.
switch event.Type {
case "tool_start":
if event.ToolName != "" {
p.streamCurrentTool = event.ToolName
}
case "tool_end":
p.streamCurrentTool = ""
}
// Bound payload sizes so streaming and replay buffers can't balloon due to a single tool
// output or oversized content chunk.
event = truncateStreamEvent(event)
// Decorate once so every subscriber sees identical meta.
event = p.decorateStreamEventLocked(event)
p.appendStreamEventLocked(event)
var staleChannels []chan PatrolStreamEvent
dropReasons := make(map[chan PatrolStreamEvent]string)
for ch, sub := range p.streamSubscribers {
if sub == nil || sub.closed.Load() {
staleChannels = append(staleChannels, ch)
dropReasons[ch] = "closed"
continue
}
select {
case ch <- event:
// Successfully sent
sub.fullCount = 0
default:
// Channel full. Tolerate bursts, but disconnect subscribers that are
// consistently unable to receive events (likely dead/slow clients).
sub.fullCount++
if sub.fullCount >= 25 {
staleChannels = append(staleChannels, ch)
dropReasons[ch] = "backpressure"
}
}
}
// Clean up stale subscribers using atomic CAS for safe close
for _, ch := range staleChannels {
sub := p.streamSubscribers[ch]
delete(p.streamSubscribers, ch)
reason := dropReasons[ch]
GetPatrolMetrics().RecordStreamSubscriberDrop(reason)
log.Debug().Str("reason", reason).Msg("patrol stream subscriber dropped")
if sub != nil && sub.closed.CompareAndSwap(false, true) {
close(ch)
}
}
}
// resetStreamForRun resets stream state for a new run so late-joiners don't see stale output.
// This should only be called for runs that will actually stream events (NoStream=false).
func (p *PatrolService) resetStreamForRun(runID string) {
p.streamMu.Lock()
p.streamRunID = runID
p.streamSeq = 0
p.streamPhase = "idle"
p.streamCurrentTool = ""
p.currentOutput.Reset()
p.streamEvents = nil
p.streamMu.Unlock()
}
func (p *PatrolService) decorateStreamEventLocked(event PatrolStreamEvent) PatrolStreamEvent {
if event.RunID == "" {
event.RunID = p.streamRunID
}
if event.Seq == 0 {
p.streamSeq++
event.Seq = p.streamSeq
}
if event.TsMs == 0 {
event.TsMs = time.Now().UnixMilli()
}
return event
}
const patrolStreamReplayBufferSize = 200
const patrolStreamMaxEventFieldBytes = 8 * 1024
func truncateStreamEvent(event PatrolStreamEvent) PatrolStreamEvent {
event.Content = truncateStreamField(event.Content, patrolStreamMaxEventFieldBytes)
event.ToolInput = truncateStreamField(event.ToolInput, patrolStreamMaxEventFieldBytes)
event.ToolRawInput = truncateStreamField(event.ToolRawInput, patrolStreamMaxEventFieldBytes)
event.ToolOutput = truncateStreamField(event.ToolOutput, patrolStreamMaxEventFieldBytes)
return event
}
func truncateStreamField(s string, max int) string {
if max <= 0 || len(s) <= max {
return s
}
const suffix = "...[truncated]"
if max <= len(suffix) {
return s[:max]
}
return s[:max-len(suffix)] + suffix
}
func (p *PatrolService) appendStreamEventLocked(event PatrolStreamEvent) {
// Keep a bounded buffer for Last-Event-ID replay (best-effort).
p.streamEvents = append(p.streamEvents, event)
if len(p.streamEvents) > patrolStreamReplayBufferSize {
p.streamEvents = p.streamEvents[len(p.streamEvents)-patrolStreamReplayBufferSize:]
}
}
func (p *PatrolService) streamEventsSinceLocked(lastSeq int64) []PatrolStreamEvent {
// Seq is monotonic within a run; we reset buffer on new run.
for i := len(p.streamEvents) - 1; i >= 0; i-- {
if p.streamEvents[i].Seq <= lastSeq {
// Return events after i
out := make([]PatrolStreamEvent, len(p.streamEvents)-(i+1))
copy(out, p.streamEvents[i+1:])
return out
}
}
// All buffered events are newer
out := make([]PatrolStreamEvent, len(p.streamEvents))
copy(out, p.streamEvents)
return out
}
func (p *PatrolService) streamBufferWindowLocked() (start, end int64) {
if len(p.streamEvents) == 0 {
return 0, 0
}
return p.streamEvents[0].Seq, p.streamEvents[len(p.streamEvents)-1].Seq
}
func (p *PatrolService) makeSnapshotLocked(reason string) PatrolStreamEvent {
start, end := p.streamBufferWindowLocked()
phase := p.streamPhase
if phase == "idle" {
phase = ""
}
tr := p.currentOutput.Truncated()
var trPtr *bool
if tr {
trPtr = &tr
}
// Snapshot is synthetic and should not advance seq; use the most recent real event seq
// so clients can resume from a meaningful Last-Event-ID.
seq := end
return PatrolStreamEvent{
Type: "snapshot",
RunID: p.streamRunID,
Seq: seq,
TsMs: time.Now().UnixMilli(),
ResyncReason: reason,
BufferStart: start,
BufferEnd: end,
ContentTruncated: trPtr,
Phase: phase,
Content: p.currentOutput.String(),
ToolName: p.streamCurrentTool,
}
}
// appendStreamContent adds content to the current output and broadcasts it
func (p *PatrolService) appendStreamContent(content string) {
p.streamMu.Lock()
p.currentOutput.WriteString(content)
p.streamMu.Unlock()
p.broadcast(PatrolStreamEvent{
Type: "content",
Content: content,
})
}
// setStreamPhase updates the current phase and broadcasts it to all subscribers.
// The frontend only updates its phase display when it receives a 'phase' event,
// so we must broadcast phase changes to keep the UI in sync.
func (p *PatrolService) setStreamPhase(phase string) {
p.streamMu.Lock()
oldPhase := p.streamPhase
p.streamPhase = phase
p.streamMu.Unlock()
// Broadcast phase change (except for idle which just clears state)
// This ensures late joiners and continuous watchers see the current phase
if phase != "idle" && phase != oldPhase {
p.broadcast(PatrolStreamEvent{
Type: "phase",
Phase: phase,
})
}
}
// GetCurrentStreamOutput returns the current buffered output (for late joiners)
func (p *PatrolService) GetCurrentStreamOutput() (string, string) {
p.streamMu.RLock()
defer p.streamMu.RUnlock()
return p.currentOutput.String(), p.streamPhase
}
// reviewAndResolveAlerts uses AI to review active alerts and resolve those where the issue is fixed.
// This is the core of autonomous alert management - the AI looks at each alert, checks current state,
// and determines if the underlying issue has been resolved.
//
// Two structural cost controls run before any model call:
// - alerts whose trigger condition demonstrably still holds in the current
// snapshot (metric still at/above threshold, resource still offline,
// Docker container still unhealthy, stopped or OOM-killed) are skipped —
// a correct review could only answer KEEP, and a wrong RESOLVE would be
// raised again by the alert engine on the resource's next report. The
// question is not worth a billed call. Resolution stays model-owned:
// Pulse never locally decides that an issue HAS cleared.
// - the remaining candidates are reviewed in batched model calls instead
// of one call per alert.
func (p *PatrolService) reviewAndResolveAlertsState(ctx context.Context, state patrolRuntimeState, llmAllowed bool, executionID string) int {
p.mu.RLock()
resolver := p.alertResolver
aiService := p.aiService
p.mu.RUnlock()
if resolver == nil {
return 0
}
activeAlerts := resolver.GetActiveAlerts()
if len(activeAlerts) == 0 {
return 0
}
aiSvc := aiService
if !llmAllowed {
aiSvc = nil
}
if aiSvc == nil || !aiSvc.IsEnabled() {
return 0
}
if state.alertResourceResolver == nil {
// Built once: each review context may consult it.
state.alertResourceResolver = patrolAlertReferenceResolver(state)
}
// Only review alerts that have been active for at least 10 minutes
// This avoids thrashing on transient alerts
minAge := 10 * time.Minute
var alertsToReview []AlertInfo
stillFiring := 0
for _, alert := range activeAlerts {
if time.Since(alert.StartTime) < minAge {
continue
}
if p.alertStillFiringState(alert, state) {
stillFiring++
continue
}
alertsToReview = append(alertsToReview, alert)
}
if len(alertsToReview) == 0 {
if stillFiring > 0 {
log.Debug().
Int("still_firing", stillFiring).
Msg("AI Patrol: All aged alerts still demonstrably firing, skipping model review")
}
return 0
}
log.Info().
Int("total_active", len(activeAlerts)).
Int("to_review", len(alertsToReview)).
Int("still_firing_skipped", stillFiring).
Msg("AI Patrol: Reviewing alerts for auto-resolution")
resolvedCount := 0
for _, verdict := range p.reviewAlertsWithModelState(ctx, alertsToReview, state, aiSvc, executionID) {
if !verdict.resolve {
continue
}
if resolver.ResolveAlert(verdict.alert.ID) {
resolvedCount++
log.Info().
Str("alertID", verdict.alert.ID).
Str("resource", verdict.alert.ResourceName).
Str("reason", verdict.reason).
Dur("age", time.Since(verdict.alert.StartTime)).
Msg("AI Patrol: Auto-resolved alert - issue no longer detected")
}
}
if resolvedCount > 0 {
log.Info().
Int("resolved", resolvedCount).
Msg("AI Patrol: Completed alert review")
}
return resolvedCount
}
// alertStillFiringState reports whether the current runtime snapshot still
// demonstrably shows the alert's trigger condition — a metric alert whose
// current value is still at or above its threshold, an offline alert
// whose resource is still not online, or a Docker container alert whose
// container still reports the state that raised it (for a restart loop,
// a container caught in Docker's restart backoff). For these alerts a
// correct model review can only answer KEEP, so Patrol skips the billed
// call and keeps the alert. The gate never decides that an issue HAS
// cleared: anything uncertain (unknown alert type, missing resource,
// unmapped metric) returns false so the model still owns the resolution
// decision.
func (p *PatrolService) alertStillFiringState(alert AlertInfo, snap patrolRuntimeState) bool {
switch alert.Type {
case "cpu", "memory", "usage", "disk":
if alert.Threshold <= 0 {
return false
}
// getCurrentMetricValueState returns -1 when the resource or metric
// is unknown, which can never satisfy a positive threshold.
return p.getCurrentMetricValueState(alert, snap) >= alert.Threshold
case "offline", "powered-off", "host-offline":
resource := lookupPatrolAlertResourceState(alert, snap)
if !resource.found {
// Resource is gone from the snapshot — the model should judge
// whether the alert resolves as removed.
return false
}
switch resource.resourceType {
case "node", "agent", "vm", "system-container":
return !p.isResourceOnlineState(alert, snap)
default:
return false
}
case "docker-container-health", "docker-container-state", "docker-container-oom-kill", "docker-container-restart-loop":
resource := lookupPatrolAlertResourceState(alert, snap)
if !resource.found || resource.resourceType != "app-container" {
return false
}
return dockerContainerAlertStillFiring(alert.Type, resource)
default:
return false
}
}
// dockerContainerAlertingStates are the runtime states the alert subsystem
// raises docker-container-state for (checkDockerContainerState).
var dockerContainerAlertingStates = map[string]bool{
"created": true, "restarting": true, "removing": true, "paused": true,
"exited": true, "dead": true, "unknown": true,
}
// dockerContainerAlertStillFiring mirrors the alert subsystem's Docker
// container health, state and OOM checks (internal/alerts/docker.go)
// against the container's current report. It answers only in the keep
// direction: a container that may have recovered returns false and goes
// to the model.
func dockerContainerAlertStillFiring(alertType string, container patrolAlertResourceState) bool {
state := strings.ToLower(strings.TrimSpace(container.status))
switch alertType {
case "docker-container-health":
// Health is live evidence only while the container runs; a
// stopped container's last health result belongs to the state
// alert, which the alert subsystem raises instead.
if state != "" && state != "running" {
return false
}
switch strings.ToLower(strings.TrimSpace(container.health)) {
case "", "none", "healthy", "starting":
return false
}
return true
case "docker-container-state":
return dockerContainerAlertingStates[state]
case "docker-container-oom-kill":
return (state == "exited" || state == "dead") && container.oomKilled != nil && *container.oomKilled
case "docker-container-restart-loop":
// The alert subsystem counts restarts over a window the snapshot
// does not carry, so this is not its check. A container caught in
// Docker's restart backoff is still restarting, though; keeping
// its alert defers to the alert subsystem, which clears it once
// the window empties.
return state == "restarting"
default:
return false
}
}
// getCurrentMetricValue gets the current value of the metric that triggered the alert
type patrolAlertResourceState struct {
resourceType string
platform string
name string
status string
health string
restartCount int
oomKilled *bool
cpu float64
memory float64
guestMemory *patrolGuestMemoryReading
disk float64
found bool
}
func patrolAlertNameMatches(alert AlertInfo, ids ...string) bool {
for _, id := range ids {
if strings.TrimSpace(id) == "" {
continue
}
if id == alert.ResourceID || id == alert.ResourceName {
return true
}
}
return false
}
func patrolAlertLookupType(alert AlertInfo) string {
resourceType := canonicalizeAICompatibilityResourceType(alert.ResourceType)
if alert.Type == "usage" && (resourceType == "" || strings.EqualFold(resourceType, "usage")) {
return "storage"
}
return resourceType
}
func lookupPatrolAlertResourceState(alert AlertInfo, snap patrolRuntimeState) patrolAlertResourceState {
alert.ResourceType = patrolAlertLookupType(alert)
if alert.ResourceType == "app-container" {
if resource, ok := patrolLookupAppContainerAlertResourceState(alert, snap); ok {
return resource
}
}
switch alert.ResourceType {
case "storage":
if resource, ok := patrolLookupStorageAlertResourceState(alert, snap); ok {
return resource
}
case "node":
if resource, ok := patrolLookupNodeAlertResourceState(alert, snap); ok {
return resource
}
case "vm":
if resource, ok := patrolLookupGuestAlertResourceState(alert, snap, "VM", "vm"); ok {
return resource
}
case "system-container":
if resource, ok := patrolLookupGuestAlertResourceState(alert, snap, "Container", "system-container"); ok {
return resource
}
case "agent":
if resource, ok := patrolLookupHostAlertResourceState(alert, snap); ok {
return resource
}
}
return patrolAlertResourceState{}
}
// patrolAlertSubjectInInventory reports whether current inventory holds the
// alert's subject, or the resource that owns it, by durable identity.
func patrolAlertSubjectInInventory(alert AlertInfo, snap patrolRuntimeState) bool {
if strings.TrimSpace(alert.ResourceID) == "" {
return false
}
binding, _ := patrolAlertReferenceResolver(snap).ResolveAlertResourceReference(alert.ResourceID)
return binding.ResourceID != ""
}
func patrolLookupAppContainerAlertResourceState(alert AlertInfo, snap patrolRuntimeState) (patrolAlertResourceState, bool) {
rows := patrolAppContainerRows(snap, nil)
// An alert subsystem container ID ("docker:<host>/<container>") names
// one container on one host, so it alone decides: a display-name match
// could judge the alert against a same-named container elsewhere.
exact := strings.HasPrefix(alert.ResourceID, "docker:")
for _, container := range rows {
if exact {
if container.alertResourceID != alert.ResourceID {
continue
}
} else if !patrolAlertNameMatches(alert, container.id, container.name) {
continue
}
return patrolAlertResourceState{
resourceType: "app-container",
name: container.name,
status: container.status,
health: container.health,
restartCount: container.restartCount,
oomKilled: container.oomKilled,
cpu: container.cpu,
memory: container.memory,
found: true,
}, true
}
return patrolAlertResourceState{}, false
}
func patrolLookupStorageAlertResourceState(alert AlertInfo, snap patrolRuntimeState) (patrolAlertResourceState, bool) {
for _, storage := range patrolStoragePoolRows(snap, nil) {
if !patrolAlertNameMatches(alert, storage.id, storage.name) {
continue
}
return patrolAlertResourceState{
resourceType: "storage",
name: storage.name,
status: storage.status,
disk: storage.usage,
found: true,
}, true
}
return patrolAlertResourceState{}, false
}
func patrolLookupNodeAlertResourceState(alert AlertInfo, snap patrolRuntimeState) (patrolAlertResourceState, bool) {
for _, node := range patrolNodeInventoryRows(snap, nil) {
if !patrolAlertNameMatches(alert, node.id, node.name) {
continue
}
return patrolAlertResourceState{
resourceType: "node",
name: node.name,
status: node.status,
cpu: node.cpu,
memory: node.mem,
found: true,
}, true
}
return patrolAlertResourceState{}, false
}
func patrolLookupGuestAlertResourceState(alert AlertInfo, snap patrolRuntimeState, guestType, resourceType string) (patrolAlertResourceState, bool) {
for _, guest := range patrolGuestInventoryRows(snap, nil, nil) {
if guest.gType != guestType || !patrolAlertNameMatches(alert, guest.id, guest.name) {
continue
}
return patrolAlertResourceState{
resourceType: resourceType,
name: guest.name,
status: guest.status,
cpu: guest.cpu,
memory: guest.mem,
guestMemory: &guest.memory,
found: true,
}, true
}
return patrolAlertResourceState{}, false
}
type patrolHostAlertRow struct {
id, name, hostname, status, platform string
cpu, memory float64
}
func patrolHostAlertRows(snap patrolRuntimeState) []patrolHostAlertRow {
if snap.readState != nil {
hosts := snap.readState.Hosts()
rows := make([]patrolHostAlertRow, 0, len(hosts))
for _, host := range hosts {
rows = append(rows, patrolHostAlertRow{
id: host.ID(),
name: host.Name(),
hostname: host.Hostname(),
status: string(host.Status()),
platform: host.Platform(),
cpu: host.CPUPercent(),
memory: host.MemoryPercent(),
})
}
return rows
}
rows := make([]patrolHostAlertRow, 0, len(snap.Hosts))
for _, host := range snap.Hosts {
rows = append(rows, patrolHostAlertRow{
id: host.ID,
name: host.DisplayName,
hostname: host.Hostname,
status: host.Status,
platform: host.Platform,
cpu: host.CPUUsage,
memory: host.Memory.Usage,
})
}
return rows
}
func patrolHostResourceType(platform string) string {
return "agent"
}
func patrolLookupHostAlertResourceState(alert AlertInfo, snap patrolRuntimeState) (patrolAlertResourceState, bool) {
for _, host := range patrolHostAlertRows(snap) {
if !patrolAlertNameMatches(alert, host.id, host.name, host.hostname) {
continue
}
name := host.hostname
if name == "" {
name = host.name
}
return patrolAlertResourceState{
resourceType: patrolHostResourceType(host.platform),
platform: host.platform,
name: name,
status: host.status,
cpu: host.cpu,
memory: host.memory,
found: true,
}, true
}
return patrolAlertResourceState{}, false
}
func (p *PatrolService) getCurrentMetricValueState(alert AlertInfo, snap patrolRuntimeState) float64 {
resource := lookupPatrolAlertResourceState(alert, snap)
if !resource.found {
return -1
}
switch alert.Type {
case "cpu":
return resource.cpu
case "memory":
if resource.guestMemory != nil && !resource.guestMemory.pressureKnown {
return -1
}
return resource.memory
default:
if resource.resourceType == "storage" {
return resource.disk
}
}
return -1
}
// isResourceOnline checks if a resource that triggered an offline alert is now online
func (p *PatrolService) isResourceOnlineState(alert AlertInfo, snap patrolRuntimeState) bool {
resource := lookupPatrolAlertResourceState(alert, snap)
if !resource.found {
return false
}
switch resource.resourceType {
case "node", "agent":
return resource.status == "online"
case "vm", "system-container", "app-container":
return resource.status == "running" || resource.status == string(unifiedresources.StatusOnline)
default:
return false
}
}
// patrolAlertReviewBatchSize caps how many alerts are reviewed per model
// call so a large alert backlog cannot build an unbounded prompt.
const patrolAlertReviewBatchSize = 20
type patrolAlertReviewVerdict struct {
alert AlertInfo
resolve bool
reason string
}
// patrolAlertVerdictLine matches one per-alert verdict line in a batched
// alert-review model response, e.g. "ALERT 2: RESOLVE: usage back to 40%".
var patrolAlertVerdictLine = regexp.MustCompile(`(?im)^\s*ALERT\s+(\d+)\s*:\s*(RESOLVE|KEEP)\b\s*:?\s*(.*)$`)
// reviewAlertsWithModelState reviews the candidate alerts with the model in
// batched calls and returns one verdict per alert, in order. Alerts whose
// verdict cannot be parsed from the model response default to KEEP.
func (p *PatrolService) reviewAlertsWithModelState(ctx context.Context, candidates []AlertInfo, snap patrolRuntimeState, aiService *Service, executionID string) []patrolAlertReviewVerdict {
verdicts := make([]patrolAlertReviewVerdict, 0, len(candidates))
for start := 0; start < len(candidates); start += patrolAlertReviewBatchSize {
end := min(start+patrolAlertReviewBatchSize, len(candidates))
verdicts = append(verdicts, p.reviewAlertBatchState(ctx, candidates[start:end], snap, aiService, executionID)...)
}
return verdicts
}
// reviewAlertBatchState asks the model for a RESOLVE/KEEP verdict on each
// alert in one call. Any error or unparseable verdict keeps the alert —
// the fail-safe direction matches the single-alert behavior this replaces.
func (p *PatrolService) reviewAlertBatchState(ctx context.Context, batch []AlertInfo, snap patrolRuntimeState, aiService *Service, executionID string) []patrolAlertReviewVerdict {
verdicts := make([]patrolAlertReviewVerdict, len(batch))
for i, alert := range batch {
verdicts[i] = patrolAlertReviewVerdict{alert: alert}
}
// Use a quick, low-cost AI call
response, err := aiService.QuickAnalysis(ctx, QuickAnalysisRequest{
Prompt: p.buildAlertReviewPromptState(batch, snap),
ExecutionID: executionID,
UseCase: "patrol",
TargetType: "alert_autoresolve",
})
if err != nil {
log.Debug().Err(err).Int("alerts", len(batch)).Msg("AI Patrol: Failed to get AI judgment on alert batch")
return verdicts
}
matched := applyAlertReviewResponse(verdicts, response)
// Single-alert fallback: accept a bare "RESOLVE: <reason>" response
// without the "ALERT 1:" prefix, matching the pre-batch prompt contract.
if len(batch) == 1 && !matched {
trimmed := strings.TrimSpace(response)
if strings.HasPrefix(strings.ToUpper(trimmed), "RESOLVE:") {
verdicts[0].resolve = true
verdicts[0].reason = patrolAlertResolveReason(strings.TrimPrefix(trimmed[len("RESOLVE:"):], " "))
}
}
// This quick review has only the supplied snapshot, not an independent
// guest measurement. Missing/stale/cache-inclusive memory cannot prove a
// memory alert recovered, even if the model asks to resolve it.
for i, alert := range batch {
// The snapshot carries no native sensor readings, so nothing in it
// can show that one sensor's temperature recovered.
if isNativeSensorAlertResource(alert.ResourceID) {
verdicts[i].resolve = false
verdicts[i].reason = ""
continue
}
if alert.Type != "memory" {
continue
}
resource := lookupPatrolAlertResourceState(alert, snap)
if resource.guestMemory != nil && !resource.guestMemory.pressureKnown {
verdicts[i].resolve = false
verdicts[i].reason = ""
}
}
return verdicts
}
// isNativeSensorAlertResource reports whether an alert names one native
// sensor of a host agent ("agent:<host>/sensor:<hex>").
func isNativeSensorAlertResource(resourceID string) bool {
rest, ok := strings.CutPrefix(strings.TrimSpace(resourceID), "agent:")
if !ok {
return false
}
_, child, ok := strings.Cut(rest, "/")
return ok && strings.HasPrefix(child, "sensor:")
}
// buildAlertReviewPromptState formats the batched alert-review prompt: one
// numbered section per alert with its trigger details and current resource
// state, plus strict per-line verdict output instructions. The current
// state covers only some alert conditions, so the prompt says that a
// condition it does not show is a KEEP, not a recovery.
func (p *PatrolService) buildAlertReviewPromptState(batch []AlertInfo, snap patrolRuntimeState) string {
var sb strings.Builder
sb.WriteString("Review these active alerts and determine, for each, if it should be auto-resolved based on current state.\n")
for i, alert := range batch {
alertType := patrolAlertLookupType(alert)
sb.WriteString(fmt.Sprintf(`
ALERT %d:
- ID: %s
- Type: %s
- Resource: %s (%s)
- Message: %s
- Value when triggered: %.1f
- Threshold: %.1f
- Active for: %s
CURRENT STATE OF THIS RESOURCE:
%s
`,
i+1, alert.ID, alert.Type, alert.ResourceName, alertType,
alert.Message, alert.Value, alert.Threshold, alert.Duration,
p.getResourceCurrentStateState(alert, snap)))
}
sb.WriteString(`
For each alert, should it be RESOLVED because the underlying issue is fixed?
Answer RESOLVE only when the current state shown for that alert directly shows its condition has cleared (for example the metric is back under its threshold, the resource is online again, or the reported health is healthy), or shows the resource no longer exists. If the current state does not cover the condition the alert is about, answer KEEP: normal readings for other metrics are not evidence that it is fixed.
Respond with EXACTLY one line per alert, in order, and nothing else:
ALERT <number>: RESOLVE: <brief reason>
or
ALERT <number>: KEEP: <brief reason>`)
return sb.String()
}
// applyAlertReviewResponse parses per-alert verdict lines from the model
// response into verdicts. The first line seen for each alert number wins;
// out-of-range numbers are ignored. Returns whether any line matched.
func applyAlertReviewResponse(verdicts []patrolAlertReviewVerdict, response string) bool {
matched := false
seen := make(map[int]bool)
for _, m := range patrolAlertVerdictLine.FindAllStringSubmatch(response, -1) {
idx, err := strconv.Atoi(m[1])
if err != nil || idx < 1 || idx > len(verdicts) || seen[idx] {
continue
}
seen[idx] = true
matched = true
if strings.EqualFold(m[2], "RESOLVE") {
verdicts[idx-1].resolve = true
verdicts[idx-1].reason = patrolAlertResolveReason(m[3])
}
}
return matched
}
func patrolAlertResolveReason(raw string) string {
reason := strings.TrimSpace(raw)
if reason == "" {
reason = "issue no longer detected"
}
return "Patrol: " + reason
}
// getResourceCurrentState returns a description of the resource's current state
func (p *PatrolService) getResourceCurrentStateState(alert AlertInfo, snap patrolRuntimeState) string {
resource := lookupPatrolAlertResourceState(alert, snap)
if !resource.found {
// The lookup reads only some alert subjects. One the canonical
// inventory still holds, or whose owner it holds (a ZFS pool's
// storage, a guest's backup, a Swarm service), is not missing, and
// calling it missing invites the model to resolve it as removed.
if patrolAlertSubjectInInventory(alert, snap) {
return "Present in current inventory (this alert's resource or the resource that owns it); this review has no current reading of the alert's condition"
}
switch patrolAlertLookupType(alert) {
case "storage":
return "Storage not found in current state (may have been removed)"
case "node":
return "Node not found in current state"
case "agent":
return "Agent host not found in current state"
case "vm":
return "VM not found in current state"
case "system-container":
return "Container not found in current state"
case "app-container":
return "Docker container not found in current state"
default:
return "Resource state unknown"
}
}
switch resource.resourceType {
case "storage":
return fmt.Sprintf("Storage '%s': %.1f%% used, status: %s", resource.name, resource.disk, resource.status)
case "node":
return fmt.Sprintf("Node '%s': CPU %.1f%%, Memory %.1f%%, Status: %s",
resource.name, resource.cpu, resource.memory, resource.status)
case "agent":
if strings.EqualFold(strings.TrimSpace(resource.platform), "truenas") {
return fmt.Sprintf("TrueNAS system '%s': CPU %.1f%%, Memory %.1f%%, Status: %s",
resource.name, resource.cpu, resource.memory, resource.status)
}
return fmt.Sprintf("Agent host '%s': CPU %.1f%%, Memory %.1f%%, Status: %s",
resource.name, resource.cpu, resource.memory, resource.status)
case "vm":
return fmt.Sprintf("VM '%s': CPU %.1f%%, Memory %s, Status: %s",
resource.name, resource.cpu, resource.guestMemory.display(), resource.status)
case "system-container":
return fmt.Sprintf("Container '%s': CPU %.1f%%, Memory %s, Status: %s",
resource.name, resource.cpu, resource.guestMemory.display(), resource.status)
case "app-container":
return fmt.Sprintf("Docker container '%s': CPU %.1f%%, Memory %.1f%%, State: %s, Health: %s, Restart count: %d, OOM-killed: %s",
resource.name, resource.cpu, resource.memory, resource.status,
dockerContainerHealthText(resource.health), resource.restartCount, dockerContainerOOMText(resource.oomKilled))
default:
return "Resource state unknown"
}
}
// dockerContainerHealthText names the container's reported health check
// result, so a health alert's review sees the reading that raised it.
func dockerContainerHealthText(health string) string {
if health = strings.TrimSpace(health); health == "" {
return "none"
}
return health
}
// dockerContainerOOMText keeps an explicit "no" apart from an agent that
// does not report OOM kills, which proves nothing either way.
func dockerContainerOOMText(oomKilled *bool) string {
switch {
case oomKilled == nil:
return "unknown"
case *oomKilled:
return "yes"
default:
return "no"
}
}
// TriggerPatrolForAlert triggers an immediate patrol for a specific alert
func (p *PatrolService) TriggerPatrolForAlert(alert *alerts.Alert) {
if alert == nil {
return
}
p.mu.RLock()
triggerManager := p.triggerManager
eventTriggerConfig := p.eventTriggerConfig
eventTriggerBlock := p.eventTriggerBlock
p.mu.RUnlock()
// Gate: skip if alert-driven scoped patrols are disabled
if !eventTriggerConfig.AlertTriggersEnabled {
log.Debug().
Str("alert_identifier", alert.ID).
Msg("alert-triggered patrol skipped: alert trigger source disabled")
return
}
if eventTriggerBlock.active() {
log.Debug().
Str("alert_identifier", alert.ID).
Str("block_reason", eventTriggerBlock.Reason).
Msg("alert-triggered patrol skipped: event triggers blocked by runtime policy")
return
}
resourceType := inferResourceType(alert.Type, alert.Metadata)
if triggerManager != nil {
scope := AlertTriggeredPatrolScope(alert.ID, alert.ResourceID, resourceType, alert.Type)
if triggerManager.TriggerPatrol(scope) {
log.Debug().Str("alert_identifier", alert.ID).Msg("queued alert-triggered patrol via trigger manager")
} else {
log.Warn().Str("alert_identifier", alert.ID).Msg("alert-triggered patrol rejected by trigger manager")
}
return
}
// Non-blocking send
select {
case p.adHocTrigger <- alert:
log.Debug().Str("alert_identifier", alert.ID).Msg("queued ad-hoc patrol trigger")
default:
log.Warn().Str("alert_identifier", alert.ID).Msg("patrol trigger queue full, dropping trigger")
}
}
func (p *PatrolService) beginRun(kind string) (*patrolRunStart, bool) {
p.mu.Lock()
if p.runInProgress {
// Detect stuck runs: if the current run has been going for >20 minutes,
// force-clear the flag so a new run can proceed.
if !p.runStartedAt.IsZero() && time.Since(p.runStartedAt) > 20*time.Minute {
log.Warn().
Str("kind", kind).
Time("started_at", p.runStartedAt).
Dur("elapsed", time.Since(p.runStartedAt)).
Msg("AI Patrol: Previous run appears stuck (>20min), force-clearing runInProgress")
p.runInProgress = false
// Fall through to start new run
} else {
p.mu.Unlock()
if kind == "scoped" {
GetPatrolMetrics().RecordScopedDropped()
log.Warn().Str("kind", kind).Msg("AI Patrol: Run already in progress, dropping scoped patrol")
} else {
log.Debug().Str("kind", kind).Msg("AI Patrol: Run already in progress, skipping")
}
return nil, false
}
}
startedAt := time.Now()
runID := fmt.Sprintf("%d", startedAt.UnixNano())
p.runInProgress = true
p.currentRunID = runID
p.runStartedAt = startedAt
p.mu.Unlock()
return &patrolRunStart{id: runID, startedAt: startedAt}, true
}
func (p *PatrolService) tryStartRun(kind string) bool {
_, accepted := p.beginRun(kind)
return accepted
}
func (p *PatrolService) endRun() {
p.mu.Lock()
p.runInProgress = false
p.currentRunID = ""
p.runStartedAt = time.Time{}
orch := p.investigationOrchestrator
p.mu.Unlock()
// Periodic investigation store maintenance after each run
if maintainer, ok := orch.(InvestigationStoreMaintainer); ok {
maintainer.CleanupInvestigationStore(24*time.Hour, 1000)
}
}
// runTargetedPatrol executes a focused patrol for a specific alert
func (p *PatrolService) runTargetedPatrol(ctx context.Context, alert *alerts.Alert) {
log.Info().
Str("alert_identifier", alert.ID).
Str("resource_id", alert.ResourceID).
Msg("Running targeted AI patrol for alert")
resourceType := inferResourceType(alert.Type, alert.Metadata)
scope := AlertTriggeredPatrolScope(alert.ID, alert.ResourceID, resourceType, alert.Type)
p.TriggerScopedPatrol(ctx, scope)
}
// joinParts joins string parts with commas and "and" for the last element
func joinParts(parts []string) string {
if len(parts) == 0 {
return ""
}
if len(parts) == 1 {
return parts[0]
}
if len(parts) == 2 {
return parts[0] + " and " + parts[1]
}
return strings.Join(parts[:len(parts)-1], ", ") + ", and " + parts[len(parts)-1]
}
// generateFindingID creates a stable ID for a finding based on resource, category, and issue.
// All three components are included to ensure distinct issues on the same resource remain separate.
func generateFindingID(resourceID, category, issue string) string {
hash := sha256.Sum256([]byte(fmt.Sprintf("%s:%s:%s", resourceID, category, issue)))
return fmt.Sprintf("%x", hash[:8])
}