Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
// Package ai provides AI-powered infrastructure monitoring and investigation.
package ai
import (
"encoding/json"
"fmt"
"strconv"
"strings"
"sync"
"time"
"github.com/rs/zerolog/log"
"github.com/rcourtman/pulse-go-rewrite/pkg/aicontracts"
)
// Investigation limits for automatic finding investigation.
const (
investigationCooldown = 1 * time.Hour // Minimum time between investigation attempts
investigationTimeoutCooldown = 10 * time.Minute // Shorter cooldown for timeout failures
investigationFixRetryCooldown = 20 * time.Minute // Retry faster after failed fix attempts
maxInvestigationAttempts = 3 // Maximum number of investigation retries
// How long occurrence starts are kept. StartInvestigation refuses a run
// whose session is older, so a forgotten start cannot let one in.
occurrenceStartRetention = 24 * time.Hour
)
// FindingSeverity represents how urgent a finding is
type FindingSeverity string
const (
// FindingSeverityInfo is informational - user may want to know
FindingSeverityInfo FindingSeverity = "info"
// FindingSeverityWatch means something to keep an eye on
FindingSeverityWatch FindingSeverity = "watch"
// FindingSeverityWarning means action should be taken soon
FindingSeverityWarning FindingSeverity = "warning"
// FindingSeverityCritical means immediate action needed
FindingSeverityCritical FindingSeverity = "critical"
)
// FindingCategory groups findings by type
type FindingCategory string
const (
FindingCategoryPerformance FindingCategory = "performance"
FindingCategoryCapacity FindingCategory = "capacity"
FindingCategoryReliability FindingCategory = "reliability"
FindingCategoryBackup FindingCategory = "backup"
FindingCategorySecurity FindingCategory = "security"
FindingCategoryGeneral FindingCategory = "general"
)
// CategorySupportsStaleAutoResolve reports whether a finding category
// represents a continuous current-state condition that Patrol can soundly
// auto-resolve from absence in a successful run. Only `performance` and
// `capacity` qualify: those are metric thresholds (CPU above X, disk above
// Y) where the most recent scan's observation is authoritative — if the
// threshold isn't tripped now, the condition has cleared.
//
// All other categories represent discrete events or persistent states:
// `reliability` (a service that crashed, a restart loop), `backup` (a
// backup task that failed), `security` (a vulnerability that was found),
// and `general` (catch-all). For these, absence in a Patrol report only
// means the LLM didn't re-mention the finding — it does not mean the
// underlying issue has been addressed. Auto-resolving them on absence
// produced false `auto_resolved` events that the next run re-detected as
// regressions, inflating the regression counter and polluting the trust
// strip with fictional "auto-resolved" credit.
//
// Such findings stay active until explicitly resolved — either by the LLM
// calling `patrol_resolve_finding` with evidence, or by operator action.
func CategorySupportsStaleAutoResolve(category FindingCategory) bool {
switch category {
case FindingCategoryPerformance, FindingCategoryCapacity:
return true
default:
return false
}
}
// findingIdentityShiftMaxTitleOverlap is the keyword-overlap (Jaccard) bar
// below which a same-ID re-detection's title counts as a substantially
// different report (a key collision) rather than a rephrasing of the same
// issue. Rephrasings share core keywords (resource names, symptom nouns) and
// land well above this; genuinely distinct issues rarely share any.
const findingIdentityShiftMaxTitleOverlap = 0.2
// DismissReasonWillFixLater marks a finding as "I will fix this later" — an
// operational commitment with an implicit deadline. See Finding.RemindAt.
const DismissReasonWillFixLater = "will_fix_later"
// DefaultWillFixLaterRemindAfter is how long Pulse waits before nagging the
// operator about an open will_fix_later finding. This makes the dismissal a
// real commitment rather than a silent shut-up: if the finding still trips
// after this window, it surfaces again with a "reminded" lifecycle event
// instead of being swallowed forever.
const DefaultWillFixLaterRemindAfter = 7 * 24 * time.Hour
// FindingLoopState represents the current stage of the patrol detect/remediate loop.
type FindingLoopState string
const (
FindingLoopStateDetected FindingLoopState = "detected"
FindingLoopStateInvestigating FindingLoopState = "investigating"
FindingLoopStateRemediationPlanned FindingLoopState = "remediation_planned"
FindingLoopStateRemediating FindingLoopState = "remediating"
FindingLoopStateRemediationFailed FindingLoopState = "remediation_failed"
FindingLoopStateNeedsAttention FindingLoopState = "needs_attention"
FindingLoopStateTimedOut FindingLoopState = "timed_out"
FindingLoopStateResolved FindingLoopState = "resolved"
FindingLoopStateDismissed FindingLoopState = "dismissed"
FindingLoopStateSnoozed FindingLoopState = "snoozed"
FindingLoopStateSuppressed FindingLoopState = "suppressed"
)
// InvestigationStatus represents the current investigation state of a finding
type InvestigationStatus string
const (
InvestigationStatusPending InvestigationStatus = "pending" // Queued for investigation
InvestigationStatusRunning InvestigationStatus = "running" // Currently being investigated
InvestigationStatusCompleted InvestigationStatus = "completed" // Investigation finished
InvestigationStatusFailed InvestigationStatus = "failed" // Investigation errored out
InvestigationStatusNeedsAttention InvestigationStatus = "needs_attention" // Requires user input
)
// InvestigationOutcome represents the result of an investigation
type InvestigationOutcome string
const (
InvestigationOutcomeResolved InvestigationOutcome = "resolved" // Issue was automatically fixed
InvestigationOutcomeFixQueued InvestigationOutcome = "fix_queued" // Fix identified, awaiting approval
InvestigationOutcomeFixExecuted InvestigationOutcome = "fix_executed" // Fix command executed
InvestigationOutcomeFixFailed InvestigationOutcome = "fix_failed" // Fix command failed
InvestigationOutcomeFixRejected InvestigationOutcome = "fix_rejected" // Operator rejected the queued fix
InvestigationOutcomeFixVerified InvestigationOutcome = "fix_verified" // Fix command verified successful
InvestigationOutcomeFixVerificationFailed InvestigationOutcome = "fix_verification_failed" // Fix ran but issue persists
InvestigationOutcomeFixVerificationUnknown InvestigationOutcome = "fix_verification_unknown" // Fix ran but verification was inconclusive
InvestigationOutcomeNeedsAttention InvestigationOutcome = "needs_attention" // Requires user intervention
InvestigationOutcomeCannotFix InvestigationOutcome = "cannot_fix" // AI determined it cannot fix this
InvestigationOutcomeTimedOut InvestigationOutcome = "timed_out" // Transient timeout, will retry sooner
InvestigationOutcomeUnclassified InvestigationOutcome = "unclassified" // Finished, but its conclusion could not be read
InvestigationOutcomeInterrupted InvestigationOutcome = "interrupted" // Cancelled by a restart or shutdown, will retry
)
// FindingLifecycleEvent represents an append-only log entry of finding lifecycle transitions.
// It is used to make the patrol detect/remediate/verify loop observable and debuggable in the UI.
type FindingLifecycleEvent struct {
At time.Time `json:"at"`
Type string `json:"type"`
Message string `json:"message,omitempty"`
From string `json:"from,omitempty"`
To string `json:"to,omitempty"`
Metadata map[string]string `json:"metadata,omitempty"`
}
const maxFindingLifecycleEvents = 50
// Finding represents an AI-discovered insight about infrastructure.
// findingJSON deliberately mirrors this struct field-for-field
// (marshal-mirror pattern); merging them would change the public literal API.
// TestFindingJSONMirrorStaysInSync enforces the mirror.
//
// CapacityForecast is a deterministic, backend-computed capacity trend
// attached to a finding when its resource is trending toward exhaustion.
// Unlike the model-authored prose body (Description/Impact/Recommendation),
// this is a computed fact (linear regression over recent utilization
// samples) and gives the operator a first-class urgency signal independent
// of whether the model chose to mention it: "fills in ~N days at +X%/day".
// Populated by Patrol at run time from the deterministic forecast layer and
// joined onto the finding by resource ID; nil when no trend applies.
type CapacityForecast struct {
Metric string `json:"metric,omitempty"` // e.g. "storage", "disk"
CurrentPct float64 `json:"current_pct"` // current utilization %
DailyChange float64 `json:"daily_change"` // avg % change per day (negative = declining)
DaysToFull int `json:"days_to_full"` // days to 100% at current rate; -1 if stable/declining
}
//nolint:dupl // deliberate marshal-mirror twin of findingJSON
type Finding struct {
ID string `json:"id"`
Key string `json:"key,omitempty"` // Stable issue key for runbook matching
Severity FindingSeverity `json:"severity"`
Category FindingCategory `json:"category"`
ResourceID string `json:"resource_id"`
ResourceName string `json:"resource_name"`
ResourceType string `json:"resource_type"` // node, vm, container, docker, storage, agent, pbs, agent_raid
// ResourceCriticality is the operator-set resource priority stamped
// from ResourceOperatorState. It affects Patrol attention ordering
// among same-severity findings but does not alter severity.
ResourceCriticality string `json:"resource_criticality,omitempty"`
Node string `json:"node,omitempty"`
Title string `json:"title"`
Description string `json:"description"`
Impact string `json:"impact,omitempty"` // Operator-facing consequence-if-ignored statement
// PreviousResolvedFixSummary preserves the description of the fix that
// resolved this finding the last time it was active. Captured at
// regression time before InvestigationRecord is cleared, so the next
// investigation has operational memory of what worked previously.
PreviousResolvedFixSummary string `json:"previous_resolved_fix_summary,omitempty"`
Recommendation string `json:"recommendation,omitempty"`
Evidence string `json:"evidence,omitempty"` // data/commands that led to this finding
Source string `json:"source,omitempty"` // "ai-analysis" for LLM findings, empty for rule-based
FailureCause string `json:"failure_cause,omitempty"`
DetectedAt time.Time `json:"detected_at"`
LastSeenAt time.Time `json:"last_seen_at"`
ResolvedAt *time.Time `json:"resolved_at,omitempty"`
AutoResolved bool `json:"auto_resolved"`
ResolveReason string `json:"resolve_reason,omitempty"` // Why the finding was resolved (e.g., "No longer detected by patrol")
AcknowledgedAt *time.Time `json:"acknowledged_at,omitempty"`
SnoozedUntil *time.Time `json:"snoozed_until,omitempty"` // Finding hidden until this time
// Link to alert if this finding was triggered by or attached to an alert.
AlertIdentifier string `json:"-"`
// User feedback fields - enables LLM "memory" by tracking how users respond
// This helps prevent the LLM from repeatedly raising the same dismissed issues
DismissedReason string `json:"dismissed_reason,omitempty"` // "not_an_issue", "expected_behavior", "will_fix_later"
UserNote string `json:"user_note,omitempty"` // Freeform user explanation, included in LLM context
TimesRaised int `json:"times_raised"` // How many times this finding has been detected
Suppressed bool `json:"suppressed"` // Permanently suppress similar findings for this resource
// RemindAt turns "will_fix_later" into a real operational commitment.
// When set, the dismissal stays quiet until this timestamp passes; after
// it passes, the next re-detection clears the dismissal and surfaces the
// finding again with a "reminded" lifecycle event so the operator sees
// "you said you'd fix this on <date>, it's still happening." Only
// populated for reason == "will_fix_later".
RemindAt *time.Time `json:"remind_at,omitempty"`
// RemindCount counts how many times a will_fix_later commitment has
// surfaced past its RemindAt deadline. Incremented next to each
// "reminded" lifecycle event (both the re-detection path and the
// proactive sweep). Survives the clear of RemindAt so the curator UI
// can show "reminded 3x".
RemindCount int `json:"remind_count,omitempty"`
// Investigation fields - tracks autonomous AI investigation of findings
InvestigationSessionID string `json:"investigation_session_id,omitempty"` // Chat session ID if being investigated
InvestigationStatus string `json:"investigation_status,omitempty"` // pending, running, completed, failed, needs_attention
InvestigationOutcome string `json:"investigation_outcome,omitempty"` // resolved, fix_queued, fix_executed, fix_failed, fix_rejected, fix_verified, fix_verification_failed, needs_attention, cannot_fix
LastInvestigatedAt *time.Time `json:"last_investigated_at,omitempty"` // When last investigation completed
InvestigationAttempts int `json:"investigation_attempts"` // Number of investigation attempts
InvestigationRecord *aicontracts.InvestigationRecord `json:"investigation_record,omitempty"` // Durable contextual summary for Assistant/unified intelligence
LoopState string `json:"loop_state,omitempty"` // detected, investigating, remediating, resolved, etc.
Lifecycle []FindingLifecycleEvent `json:"lifecycle,omitempty"` // Bounded, append-only lifecycle log
RegressionCount int `json:"regression_count,omitempty"` // Times the issue reappeared after resolution
LastRegressionAt *time.Time `json:"last_regression_at,omitempty"` // Timestamp of most recent regression
// CapacityForecast is the deterministic capacity trend for this finding's
// resource (when applicable). Computed by Patrol independently of the
// model prose and surfaced as a first-class urgency signal. Nil when no
// trend applies or the finding predates forecast stamping.
CapacityForecast *CapacityForecast `json:"capacity_forecast,omitempty"`
// ObjectiveContext is the most recent retained operator objective whose
// local evidence caused this finding to be assessed. It is durable context
// for investigation, not action or approval authority.
ObjectiveContext *aicontracts.PatrolObjectiveContext `json:"objective_context,omitempty"`
// Flapping is set while the finding's open/resolved state has changed
// at least findingFlapThreshold times inside findingFlapWindow. The
// store collapses the individual regressed/resolved lifecycle rows into
// one flapping row while it is set. Derived: rebuilt from the lifecycle
// on the first transition after a restart rather than persisted.
Flapping *FindingFlapping `json:"flapping,omitempty"`
// MirrorsAlertID names the active alert whose resource and condition
// this finding restates (for example a Proxmox guest stopped finding
// beside a powered-off alert). Stamped by Patrol after every run and
// cleared when the alert resolves; the attention surface folds such
// findings under the alert instead of listing them twice. The ID is the
// alert's attention item (operational record) ID, which is what clients
// join on: several alerts can share a resource and kind.
MirrorsAlertID string `json:"mirrors_alert_id,omitempty"`
// MirrorsAlertType is the alert type of MirrorsAlertID.
MirrorsAlertType string `json:"mirrors_alert_type,omitempty"`
}
// findingJSON is the marshal mirror of Finding: identical fields, but
// AlertIdentifier round-trips as "alert_identifier" in persistence while the
// public struct hides it from generic encoders with `json:"-"`. The mirror is
// enforced by TestFindingJSONMirrorStaysInSync.
//
//nolint:dupl // deliberate marshal-mirror twin of Finding
type findingJSON struct {
ID string `json:"id"`
Key string `json:"key,omitempty"`
Severity FindingSeverity `json:"severity"`
Category FindingCategory `json:"category"`
ResourceID string `json:"resource_id"`
ResourceName string `json:"resource_name"`
ResourceType string `json:"resource_type"`
ResourceCriticality string `json:"resource_criticality,omitempty"`
Node string `json:"node,omitempty"`
Title string `json:"title"`
Description string `json:"description"`
Impact string `json:"impact,omitempty"`
PreviousResolvedFixSummary string `json:"previous_resolved_fix_summary,omitempty"`
Recommendation string `json:"recommendation,omitempty"`
Evidence string `json:"evidence,omitempty"`
Source string `json:"source,omitempty"`
FailureCause string `json:"failure_cause,omitempty"`
DetectedAt time.Time `json:"detected_at"`
LastSeenAt time.Time `json:"last_seen_at"`
ResolvedAt *time.Time `json:"resolved_at,omitempty"`
AutoResolved bool `json:"auto_resolved"`
ResolveReason string `json:"resolve_reason,omitempty"`
AcknowledgedAt *time.Time `json:"acknowledged_at,omitempty"`
SnoozedUntil *time.Time `json:"snoozed_until,omitempty"`
AlertIdentifier string `json:"alert_identifier,omitempty"`
DismissedReason string `json:"dismissed_reason,omitempty"`
UserNote string `json:"user_note,omitempty"`
TimesRaised int `json:"times_raised"`
Suppressed bool `json:"suppressed"`
RemindAt *time.Time `json:"remind_at,omitempty"`
RemindCount int `json:"remind_count,omitempty"`
InvestigationSessionID string `json:"investigation_session_id,omitempty"`
InvestigationStatus string `json:"investigation_status,omitempty"`
InvestigationOutcome string `json:"investigation_outcome,omitempty"`
LastInvestigatedAt *time.Time `json:"last_investigated_at,omitempty"`
InvestigationAttempts int `json:"investigation_attempts"`
InvestigationRecord *aicontracts.InvestigationRecord `json:"investigation_record,omitempty"`
LoopState string `json:"loop_state,omitempty"`
Lifecycle []FindingLifecycleEvent `json:"lifecycle,omitempty"`
RegressionCount int `json:"regression_count,omitempty"`
LastRegressionAt *time.Time `json:"last_regression_at,omitempty"`
CapacityForecast *CapacityForecast `json:"capacity_forecast,omitempty"`
ObjectiveContext *aicontracts.PatrolObjectiveContext `json:"objective_context,omitempty"`
Flapping *FindingFlapping `json:"flapping,omitempty"`
MirrorsAlertID string `json:"mirrors_alert_id,omitempty"`
MirrorsAlertType string `json:"mirrors_alert_type,omitempty"`
}
func (f Finding) MarshalJSON() ([]byte, error) {
alertIdentifier := strings.TrimSpace(f.AlertIdentifier)
return json.Marshal(findingJSON{
ID: f.ID,
Key: f.Key,
Severity: f.Severity,
Category: f.Category,
ResourceID: f.ResourceID,
ResourceName: f.ResourceName,
ResourceType: f.ResourceType,
ResourceCriticality: normalizeResourceCriticality(f.ResourceCriticality),
Node: f.Node,
Title: f.Title,
Description: f.Description,
Impact: f.Impact,
PreviousResolvedFixSummary: f.PreviousResolvedFixSummary,
Recommendation: f.Recommendation,
Evidence: f.Evidence,
Source: f.Source,
FailureCause: f.FailureCause,
DetectedAt: f.DetectedAt,
LastSeenAt: f.LastSeenAt,
ResolvedAt: f.ResolvedAt,
AutoResolved: f.AutoResolved,
ResolveReason: f.ResolveReason,
AcknowledgedAt: f.AcknowledgedAt,
SnoozedUntil: f.SnoozedUntil,
AlertIdentifier: alertIdentifier,
DismissedReason: f.DismissedReason,
UserNote: f.UserNote,
TimesRaised: f.TimesRaised,
Suppressed: f.Suppressed,
RemindAt: f.RemindAt,
RemindCount: f.RemindCount,
InvestigationSessionID: f.InvestigationSessionID,
InvestigationStatus: f.InvestigationStatus,
InvestigationOutcome: f.InvestigationOutcome,
LastInvestigatedAt: f.LastInvestigatedAt,
InvestigationAttempts: f.InvestigationAttempts,
InvestigationRecord: f.InvestigationRecord,
LoopState: f.LoopState,
Lifecycle: f.Lifecycle,
RegressionCount: f.RegressionCount,
LastRegressionAt: f.LastRegressionAt,
CapacityForecast: f.CapacityForecast,
ObjectiveContext: clonePatrolObjectiveContext(f.ObjectiveContext),
Flapping: cloneFindingFlapping(f.Flapping),
MirrorsAlertID: f.MirrorsAlertID,
MirrorsAlertType: f.MirrorsAlertType,
})
}
func (f *Finding) UnmarshalJSON(data []byte) error {
var payload findingJSON
if err := json.Unmarshal(data, &payload); err != nil {
return err
}
*f = Finding{
ID: payload.ID,
Key: payload.Key,
Severity: payload.Severity,
Category: payload.Category,
ResourceID: payload.ResourceID,
ResourceName: payload.ResourceName,
ResourceType: payload.ResourceType,
ResourceCriticality: normalizeResourceCriticality(payload.ResourceCriticality),
Node: payload.Node,
Title: payload.Title,
Description: payload.Description,
Impact: payload.Impact,
PreviousResolvedFixSummary: payload.PreviousResolvedFixSummary,
Recommendation: payload.Recommendation,
Evidence: payload.Evidence,
Source: payload.Source,
FailureCause: payload.FailureCause,
DetectedAt: payload.DetectedAt,
LastSeenAt: payload.LastSeenAt,
ResolvedAt: payload.ResolvedAt,
AutoResolved: payload.AutoResolved,
ResolveReason: payload.ResolveReason,
AcknowledgedAt: payload.AcknowledgedAt,
SnoozedUntil: payload.SnoozedUntil,
AlertIdentifier: strings.TrimSpace(payload.AlertIdentifier),
DismissedReason: payload.DismissedReason,
UserNote: payload.UserNote,
TimesRaised: payload.TimesRaised,
Suppressed: payload.Suppressed,
RemindAt: payload.RemindAt,
RemindCount: payload.RemindCount,
InvestigationSessionID: payload.InvestigationSessionID,
InvestigationStatus: payload.InvestigationStatus,
InvestigationOutcome: payload.InvestigationOutcome,
LastInvestigatedAt: payload.LastInvestigatedAt,
InvestigationAttempts: payload.InvestigationAttempts,
InvestigationRecord: payload.InvestigationRecord,
LoopState: payload.LoopState,
Lifecycle: payload.Lifecycle,
RegressionCount: payload.RegressionCount,
LastRegressionAt: payload.LastRegressionAt,
CapacityForecast: payload.CapacityForecast,
ObjectiveContext: clonePatrolObjectiveContext(payload.ObjectiveContext),
Flapping: cloneFindingFlapping(payload.Flapping),
MirrorsAlertID: payload.MirrorsAlertID,
MirrorsAlertType: payload.MirrorsAlertType,
}
return nil
}
// IsActive returns true if the finding is still active (not resolved, not snoozed, not suppressed, not dismissed)
func (f *Finding) IsActive() bool {
return f.ResolvedAt == nil && !f.IsSnoozed() && !f.Suppressed && f.DismissedReason == ""
}
// IsDismissed returns true if the user has dismissed this finding with a reason
func (f *Finding) IsDismissed() bool {
return f.DismissedReason != ""
}
// IsSnoozed returns true if the finding is currently snoozed
func (f *Finding) IsSnoozed() bool {
return f.SnoozedUntil != nil && time.Now().Before(*f.SnoozedUntil)
}
// IsResolved returns true if the finding has been resolved (ignores snooze)
func (f *Finding) IsResolved() bool {
return f.ResolvedAt != nil
}
func investigationCooldownForOutcome(outcome string) time.Duration {
switch InvestigationOutcome(outcome) {
case InvestigationOutcomeFixVerificationUnknown:
// The next model assessment owns follow-up. Investigating an uncertain
// effect is a read, not permission to retry the infrastructure write.
return 0
case InvestigationOutcomeTimedOut:
return investigationTimeoutCooldown
case InvestigationOutcomeInterrupted:
// A restart cut the run short; nothing about the finding changed.
return 0
case InvestigationOutcomeFixFailed, InvestigationOutcomeFixVerificationFailed:
return investigationFixRetryCooldown
default:
return investigationCooldown
}
}
func (f *Finding) investigationCooldown() time.Duration {
if InvestigationStatus(f.InvestigationStatus) == InvestigationStatusFailed &&
(f.InvestigationOutcome == "" ||
InvestigationOutcome(f.InvestigationOutcome) == InvestigationOutcomeTimedOut ||
InvestigationOutcome(f.InvestigationOutcome) == InvestigationOutcomeInterrupted) {
// The run failed before producing a conclusion. The scheduled Watch
// cadence owns the next assessment; there is no additional cooldown.
// Admission, attempt limits and action authority still apply.
return 0
}
return investigationCooldownForOutcome(f.InvestigationOutcome)
}
func deriveLoopState(f *Finding) FindingLoopState {
if f == nil {
return FindingLoopStateDetected
}
switch {
case f.Suppressed:
return FindingLoopStateSuppressed
case f.ResolvedAt != nil || InvestigationOutcome(f.InvestigationOutcome) == InvestigationOutcomeFixVerified:
return FindingLoopStateResolved
case f.IsSnoozed():
return FindingLoopStateSnoozed
case f.DismissedReason != "":
return FindingLoopStateDismissed
}
switch InvestigationStatus(f.InvestigationStatus) {
case InvestigationStatusRunning:
return FindingLoopStateInvestigating
case InvestigationStatusNeedsAttention:
return FindingLoopStateNeedsAttention
}
switch InvestigationOutcome(f.InvestigationOutcome) {
case InvestigationOutcomeFixQueued:
return FindingLoopStateRemediationPlanned
case InvestigationOutcomeFixExecuted:
return FindingLoopStateRemediating
case InvestigationOutcomeFixFailed, InvestigationOutcomeFixVerificationFailed:
return FindingLoopStateRemediationFailed
case InvestigationOutcomeFixRejected, InvestigationOutcomeFixVerificationUnknown, InvestigationOutcomeNeedsAttention, InvestigationOutcomeCannotFix, InvestigationOutcomeUnclassified:
// An unclassified run still left a summary for the operator to read;
// unlike needs_attention it stays eligible for another run.
return FindingLoopStateNeedsAttention
case InvestigationOutcomeTimedOut:
return FindingLoopStateTimedOut
}
return FindingLoopStateDetected
}
func (f *Finding) syncLoopState() {
if f == nil {
return
}
f.LoopState = string(deriveLoopState(f))
}
func normalizeResourceCriticality(value string) string {
switch strings.ToLower(strings.TrimSpace(value)) {
case "high", "medium", "low":
return strings.ToLower(strings.TrimSpace(value))
default:
return ""
}
}
// ShouldInvestigate returns true if this finding should be automatically investigated
// based on autonomy level, severity, and investigation history.
// A finding will NOT be investigated if:
// - Already being investigated (status = running)
// - Investigated within the last hour (cooldown)
// - Already attempted 3 times (max attempts)
// - Severity is info/watch (not actionable)
// - Already resolved/dismissed/suppressed
func (f *Finding) ShouldInvestigate(autonomyLevel string) bool {
// Only investigate if autonomy is enabled (approval or full mode)
if autonomyLevel == "" || autonomyLevel == "monitor" {
return false
}
// Don't investigate already resolved/dismissed findings
if f.ResolvedAt != nil || f.Suppressed || f.DismissedReason != "" {
return false
}
// Don't investigate snoozed findings
if f.IsSnoozed() {
return false
}
// Don't re-investigate when a fix is queued for approval/user action.
if InvestigationOutcome(f.InvestigationOutcome) == InvestigationOutcomeFixQueued {
return false
}
// Don't auto-reinvestigate fully terminal outcomes.
if InvestigationOutcome(f.InvestigationOutcome) == InvestigationOutcomeFixVerified ||
InvestigationOutcome(f.InvestigationOutcome) == InvestigationOutcomeResolved ||
InvestigationOutcome(f.InvestigationOutcome) == InvestigationOutcomeFixRejected ||
InvestigationOutcome(f.InvestigationOutcome) == InvestigationOutcomeCannotFix ||
InvestigationOutcome(f.InvestigationOutcome) == InvestigationOutcomeNeedsAttention {
return false
}
// Only investigate warning and critical severity (info/watch are not actionable)
if f.Severity != FindingSeverityWarning && f.Severity != FindingSeverityCritical {
return false
}
// Don't re-investigate if already running
if f.InvestigationStatus == string(InvestigationStatusRunning) {
return false
}
// Don't re-investigate if at max attempts (3)
if f.InvestigationAttempts >= maxInvestigationAttempts {
return false
}
// Don't re-investigate within cooldown period
// Timeout failures use a shorter cooldown (10min) since they're transient
if f.LastInvestigatedAt != nil {
cooldown := f.investigationCooldown()
if time.Since(*f.LastInvestigatedAt) < cooldown {
return false
}
}
return true
}
func inferFindingResourceType(resourceID, resourceName string) string {
joined := strings.ToLower(strings.TrimSpace(resourceID + " " + resourceName))
switch {
case strings.Contains(joined, "pbs") || strings.Contains(joined, "backup"):
return "pbs"
case strings.Contains(joined, "storage") || strings.Contains(joined, "pool") || strings.Contains(joined, "zfs"):
return "storage"
case strings.Contains(joined, "docker"):
return "app-container"
case strings.Contains(joined, "lxc") || strings.Contains(joined, "ct") || strings.Contains(joined, "container"):
return "system-container"
case strings.Contains(joined, "vm"):
return "vm"
case strings.Contains(joined, "agent"):
return "agent"
case strings.Contains(joined, "node"):
return "node"
case strings.Contains(joined, "host"):
return "agent"
}
if hasFindingNumericSuffix(resourceID) {
return "vm"
}
return "node"
}
func canonicalFindingResourceType(resourceType string) string {
normalized := canonicalizeAICompatibilityResourceType(resourceType)
if normalized == "" {
return ""
}
if isUnsupportedLegacyAIResourceTypeToken(normalized) {
return ""
}
switch normalized {
case "vm":
return "vm"
case "system-container":
return "system-container"
case "app-container":
return "app-container"
case "docker-host":
return "docker-host"
case "k8s-cluster":
return "k8s-cluster"
case "agent":
return "agent"
case "node":
return "node"
case "physical_disk":
return "physical_disk"
default:
return normalized
}
}
func hasFindingNumericSuffix(value string) bool {
value = strings.TrimSpace(value)
if value == "" {
return false
}
sep := strings.LastIndexAny(value, ":/")
if sep >= 0 && sep+1 < len(value) {
value = value[sep+1:]
}
if value == "" {
return false
}
for _, r := range value {
if r < '0' || r > '9' {
return false
}
}
return true
}
// IsBeingInvestigated returns true if an investigation is currently in progress
func (f *Finding) IsBeingInvestigated() bool {
return f.InvestigationStatus == string(InvestigationStatusRunning)
}
// CanRetryInvestigation returns true if the finding can be re-investigated
func (f *Finding) CanRetryInvestigation() bool {
// Can't retry if at max attempts
if f.InvestigationAttempts >= maxInvestigationAttempts {
return false
}
// Can't retry if still in cooldown
// Timeout failures use a shorter cooldown (10min) since they're transient
if f.LastInvestigatedAt != nil {
cooldown := f.investigationCooldown()
if time.Since(*f.LastInvestigatedAt) < cooldown {
return false
}
}
if InvestigationOutcome(f.InvestigationOutcome) == InvestigationOutcomeFixQueued ||
InvestigationOutcome(f.InvestigationOutcome) == InvestigationOutcomeFixVerified ||
InvestigationOutcome(f.InvestigationOutcome) == InvestigationOutcomeResolved ||
InvestigationOutcome(f.InvestigationOutcome) == InvestigationOutcomeFixRejected ||
InvestigationOutcome(f.InvestigationOutcome) == InvestigationOutcomeCannotFix ||
InvestigationOutcome(f.InvestigationOutcome) == InvestigationOutcomeNeedsAttention {
return false
}
// Can't retry if currently running
if f.InvestigationStatus == string(InvestigationStatusRunning) {
return false
}
return true
}
// ToCoreFinding converts to the shared InvestigationFinding type used by the
// investigation orchestrator. Centralised here so a field added to
// InvestigationFinding causes a compile error in exactly one place.
//
// The OperationalMemory projection is populated unconditionally from
// fields the internal Finding already carries (regression count,
// previous resolved fix). OperatorContext is NOT populated here
// because that data lives in the unified-resources store, which this
// package does not import. Callers that have access to the operator
// state attach it via WithOperatorContext after conversion.
func (f *Finding) ToCoreFinding() *InvestigationFinding {
core := &InvestigationFinding{
ID: f.ID,
Key: f.Key,
Severity: string(f.Severity),
Category: string(f.Category),
ResourceID: f.ResourceID,
ResourceName: f.ResourceName,
ResourceType: f.ResourceType,
Title: f.Title,
Description: f.Description,
Recommendation: f.Recommendation,
Evidence: f.Evidence,
InvestigationSessionID: f.InvestigationSessionID,
InvestigationStatus: f.InvestigationStatus,
InvestigationOutcome: f.InvestigationOutcome,
LastInvestigatedAt: f.LastInvestigatedAt,
InvestigationAttempts: f.InvestigationAttempts,
ObjectiveContext: clonePatrolObjectiveContext(f.ObjectiveContext),
}
// Populate operational memory from fields the internal Finding
// already accumulates. Only attach when there is something
// meaningful to say — a fresh finding (no regressions, no prior
// fix, no raises beyond 1) leaves the field nil so the
// orchestrator can treat absence as "no prior history" rather
// than parsing a zero-valued struct.
if f.RegressionCount > 0 || f.PreviousResolvedFixSummary != "" || f.TimesRaised > 1 {
core.OperationalMemory = &aicontracts.FindingOperationalMemory{
RegressionCount: f.RegressionCount,
LastRegressionAt: f.LastRegressionAt,
PreviousResolvedFixSummary: f.PreviousResolvedFixSummary,
TimesRaised: f.TimesRaised,
}
}
return core
}
func clonePatrolObjectiveContext(value *aicontracts.PatrolObjectiveContext) *aicontracts.PatrolObjectiveContext {
if value == nil {
return nil
}
clone := *value
clone.ObservedResourceIDs = append([]string(nil), value.ObservedResourceIDs...)
return &clone
}
// Getter methods for aicontracts.OrchestratorAIFinding interface
func (f *Finding) GetID() string { return f.ID }
func (f *Finding) GetSeverity() string { return string(f.Severity) }
func (f *Finding) GetCategory() string { return string(f.Category) }
func (f *Finding) GetResourceID() string { return f.ResourceID }
func (f *Finding) GetResourceName() string { return f.ResourceName }
func (f *Finding) GetResourceType() string { return f.ResourceType }
func (f *Finding) GetTitle() string { return f.Title }
func (f *Finding) GetDescription() string { return f.Description }
func (f *Finding) GetRecommendation() string { return f.Recommendation }
func (f *Finding) GetEvidence() string { return f.Evidence }
func (f *Finding) GetInvestigationSessionID() string { return f.InvestigationSessionID }
func (f *Finding) GetInvestigationStatus() string { return f.InvestigationStatus }
func (f *Finding) GetInvestigationOutcome() string { return f.InvestigationOutcome }
func (f *Finding) GetLastInvestigatedAt() *time.Time { return f.LastInvestigatedAt }
func (f *Finding) GetInvestigationAttempts() int { return f.InvestigationAttempts }
func (f *Finding) GetLoopState() string { return f.LoopState }
// Setter methods for aicontracts.OrchestratorAIFinding interface
func (f *Finding) SetInvestigationSessionID(v string) { f.InvestigationSessionID = v }
func (f *Finding) SetInvestigationStatus(v string) { f.InvestigationStatus = v }
func (f *Finding) SetInvestigationOutcome(v string) { f.InvestigationOutcome = v }
func (f *Finding) SetLastInvestigatedAt(v *time.Time) { f.LastInvestigatedAt = v }
func (f *Finding) SetInvestigationAttempts(v int) { f.InvestigationAttempts = v }
func (f *Finding) SetLoopState(v string) { f.LoopState = v }
// SuppressionRule represents a user-defined rule to suppress certain AI findings
// Users can create these manually to prevent alerts before they happen
type SuppressionRule struct {
ID string `json:"id"`
ResourceID string `json:"resource_id,omitempty"` // Empty means "any resource"
ResourceName string `json:"resource_name,omitempty"` // Human-readable name for display
Category FindingCategory `json:"category,omitempty"` // Empty means "any category"
Description string `json:"description"` // User's reason, e.g., "dev VM runs hot"
DismissedReason string `json:"dismissed_reason,omitempty"` // "not_an_issue", "expected_behavior", "will_fix_later"
CreatedAt time.Time `json:"created_at"`
CreatedFrom string `json:"created_from,omitempty"` // "finding" if suppressed, "dismissed" if just dismissed, "manual" if user-created
FindingID string `json:"finding_id,omitempty"` // Original finding ID if created from dismissal
}
// FindingsPersistence interface for saving/loading findings (avoids circular imports)
type FindingsPersistence interface {
SaveFindings(findings map[string]*Finding) error
LoadFindings() (map[string]*Finding, error)
}
// FindingsPersistenceWithSuppression extends persistence to include explicit suppression rules.
// This is versioned separately to keep backwards compatibility with existing persistence adapters.
type FindingsPersistenceWithSuppression interface {
FindingsPersistence
SaveFindingsAndSuppression(findings map[string]*Finding, suppressionRules map[string]*SuppressionRule) error
LoadFindingsAndSuppression() (map[string]*Finding, map[string]*SuppressionRule, error)
}
// FindingsStore provides thread-safe storage for AI findings with optional persistence
type FindingsStore struct {
mu sync.RWMutex
findings map[string]*Finding // keyed by ID
// Index by resource for quick lookups
byResource map[string][]string // resource_id -> []finding_id
// Keep track of active findings count by severity (cached, but GetSummary calculates dynamically)
activeCounts map[FindingSeverity]int
// User-defined suppression rules (separate from dismissed findings)
suppressionRules map[string]*SuppressionRule
// When this process last saw each finding begin a new occurrence, with the
// clock's monotonic reading. Never persisted: see liveRunOccurrenceLocked.
occurrenceStarts map[string]time.Time
// Persistence layer (optional)
persistence FindingsPersistence
// Debounce save operations
saveTimer *time.Timer
savePending bool
saveDebounce time.Duration
// Error tracking for persistence failures
lastSaveError error // Last error from save operation
onSaveError func(err error) // Optional callback for save errors
lastSaveTime time.Time // Last successful save time
// Optional resource-operator-state provider — when set, the store
// consults it during Add for per-resource maintenance windows and
// other operator-set intent, auto-acknowledging new findings whose
// resource is in a maintenance window. Nil-safe; an unset provider
// disables operator-state-driven suppression.
resourceOperatorStateProvider ResourceOperatorStateProvider
resourceOperatorStateProviderRevision uint64
// Optional finding-storm throttler — when set, the store invokes it
// from Add's new-finding branch so a single noisy resource's burst
// of multi-symptom findings is summarised by one storm finding.
// Nil-safe; an unset throttler disables storm summarisation.
stormThrottler *findingStormThrottler
}
// ResourceOperatorStateMaintenanceWindow describes the maintenance
// window an operator set on a resource. The findings store consults
// this through ResourceOperatorStateProvider; the package layout puts
// the canonical type in `internal/unifiedresources` to avoid an import
// cycle, so this is the projected shape the findings runtime needs.
// StartAt/EndAt bound the active occurrence (one-shot or recurring).
// Inherited marks a window stored on an ancestor whose scope covers its
// descendants; SourceID names the resource the window is stored on.
type ResourceOperatorStateMaintenanceWindow struct {
StartAt time.Time
EndAt time.Time
Reason string
SourceID string
Inherited bool
}
// ResourceOperatorStateProjection is the per-resource view the
// findings runtime consumes during the new-finding path. One provider
// call returns every signal the runtime needs, so adding new signals
// does not multiply round-trips per finding. MaintenanceWindow is nil
// when no window is currently active; IntentionallyOffline reflects
// the operator's "this resource is expected to be offline" flag.
// Criticality is an operator-set Patrol queue hint: high promotes
// same-severity findings, low demotes them, and empty means default.
type ResourceOperatorStateProjection struct {
MaintenanceWindow *ResourceOperatorStateMaintenanceWindow
IntentionallyOffline bool
MonitoringMode string
LifecycleState string
// NeverAutoRemediate is the operator's "do not act on this
// resource" flag. The findings store does not consume it during
// suppression (that's the action broker's concern), but the
// investigation runtime needs it so the orchestrator can avoid
// proposing fixes that the broker would refuse. Carrying it on
// the same projection means the suppression and investigation
// hot paths share a single read.
NeverAutoRemediate bool
Criticality string
}
// monitoringSuppressionCauseFor names the monitoring or lifecycle policy that
// silences f, or "" when none does. Retired and muted silence every finding.
// Expected-offline is availability-specific, as it is in the alert runtime: it
// silences only findings in the shared down condition class and leaves health,
// capacity, backup, and security findings visible. A projection without a
// MonitoringMode that sets only the legacy IntentionallyOffline flag is the
// pre-enum spelling of expected-offline and carries the same scope.
func (p ResourceOperatorStateProjection) monitoringSuppressionCauseFor(f *Finding) string {
if strings.EqualFold(strings.TrimSpace(p.LifecycleState), "retired") {
return "resource_retired"
}
switch strings.ToLower(strings.TrimSpace(p.MonitoringMode)) {
case "muted":
return "monitoring_muted"
case "expected_offline":
if findingConditionClass(f) == mirrorConditionDown {
return "expected_offline"
}
case "":
if p.IntentionallyOffline && findingConditionClass(f) == mirrorConditionDown {
return "intentionally_offline"
}
}
return ""
}
func (p ResourceOperatorStateProjection) suppressesFinding(f *Finding) bool {
return p.MaintenanceWindow != nil || p.monitoringSuppressionCauseFor(f) != ""
}
// ResourceOperatorStateProvider is the narrow interface the findings
// store consumes for per-resource operator intent. Implementations
// adapt the unified-resources `ResourceOperatorState` shape into the
// projection that the findings runtime needs. Returning ok=false
// means "no operator state recorded for this resource" and the store
// falls through to the default new-finding path; ok=true with all
// projection fields zero means "operator state exists but no signal
// is currently active" (e.g. window scheduled for the future).
type ResourceOperatorStateProvider interface {
OperatorStateProjection(canonicalID string, now time.Time) (ResourceOperatorStateProjection, bool)
}
// ResourceOperatorStateProviderFunc is a function adapter so wire-up
// code in the API layer can pass a closure without declaring a struct.
type ResourceOperatorStateProviderFunc func(canonicalID string, now time.Time) (ResourceOperatorStateProjection, bool)
// OperatorStateProjection implements ResourceOperatorStateProvider.
func (f ResourceOperatorStateProviderFunc) OperatorStateProjection(canonicalID string, now time.Time) (ResourceOperatorStateProjection, bool) {
if f == nil {
return ResourceOperatorStateProjection{}, false
}
return f(canonicalID, now)
}
// NewFindingsStore creates a new findings store
func NewFindingsStore() *FindingsStore {
return &FindingsStore{
findings: make(map[string]*Finding),
byResource: make(map[string][]string),
activeCounts: make(map[FindingSeverity]int),
suppressionRules: make(map[string]*SuppressionRule),
occurrenceStarts: make(map[string]time.Time),
saveDebounce: 5 * time.Second, // Debounce saves by 5 seconds
}
}
// SetResourceOperatorStateProvider installs the per-resource
// operator-state provider that the findings store consults during Add.
// Wired by the API layer at startup so the store can stay in
// `internal/ai` without importing `internal/unifiedresources`. Pass
// nil to disable operator-state-driven suppression (the default).
func (s *FindingsStore) SetResourceOperatorStateProvider(p ResourceOperatorStateProvider) {
s.mu.Lock()
defer s.mu.Unlock()
s.resourceOperatorStateProvider = p
s.resourceOperatorStateProviderRevision++
}
// SetStormThrottler installs the finding-storm throttler. The store
// consults it from Add's new-finding branch with the store mutex held;
// the throttler must not call back into the store. Pass nil to disable
// storm summarisation (the default).
func (s *FindingsStore) SetStormThrottler(t *findingStormThrottler) {
s.mu.Lock()
defer s.mu.Unlock()
s.stormThrottler = t
}
// OperatorStateProjectionFor exposes the operator-state projection for
// a resource so callers outside the suppression hot path (e.g. the
// patrol investigation runtime, when it's about to hand a finding to
// the orchestrator) can attach the same per-resource intent the
// store uses internally. Returns false when no provider is wired or
// when the provider reports no state for the resource — callers
// branch on the bool rather than parsing zero-valued projections.
func (s *FindingsStore) OperatorStateProjectionFor(resourceID string, now time.Time) (ResourceOperatorStateProjection, bool) {
s.mu.RLock()
provider := s.resourceOperatorStateProvider
s.mu.RUnlock()
if provider == nil || resourceID == "" {
return ResourceOperatorStateProjection{}, false
}
return provider.OperatorStateProjection(resourceID, now)
}
// SetPersistence sets the persistence layer and loads existing findings
func (s *FindingsStore) SetPersistence(p FindingsPersistence) error {
s.mu.Lock()
s.persistence = p
s.mu.Unlock()
// Load existing findings from disk
if p != nil {
var (
findings map[string]*Finding
rules map[string]*SuppressionRule
err error
)
if p2, ok := p.(FindingsPersistenceWithSuppression); ok {
findings, rules, err = p2.LoadFindingsAndSuppression()
} else {
findings, err = p.LoadFindings()
}
if err != nil {
return err
}
s.mu.Lock()
// Reset derived indices/caches before rehydrating.
s.byResource = make(map[string][]string)
s.activeCounts = make(map[FindingSeverity]int)
normalizedLoadedState := false
if rules != nil {
s.suppressionRules = make(map[string]*SuppressionRule, len(rules))
for id, r := range rules {
if r == nil {
continue
}
copy := *r
s.suppressionRules[id] = ©
}
}
for id, f := range findings {
if f == nil {
continue
}
if normalizeLoadedFinding(f) {
normalizedLoadedState = true
}
// One-shot cleanup of legacy alert-mirror findings. The
// deterministic detectAlertSignals → SignalActiveAlert path
// has been removed; any "Active alert detected" finding
// still persisted from an earlier build is a stale mirror
// of an alert that already owns its own canonical surface.
//
// - Active mirrors are retired (auto-resolved with a
// clear reason) so the trust strip, regression counter,
// and health score stop reflecting the duplicate while
// the operator can still see why the finding closed.
// - Already-resolved mirrors are purged entirely. They
// were never canonical Patrol output and have no
// operator value in the Resolved tab; leaving them in
// place clutters the surface and inflates the
// "regressed" totals on the trust strip.
//
// Idempotent — after the first load with this code, no
// findings match the pattern.
if legacyAlertMirrorShape(f) {
if f.ResolvedAt != nil {
// Purge: do not rehydrate. The deletion is persisted
// by the scheduleSave() call after the load loop.
normalizedLoadedState = true
continue
}
now := time.Now()
prevLoopState := f.LoopState
f.ResolvedAt = &now
f.AutoResolved = true
f.ResolveReason = "Patrol no longer mirrors alerts; the Alerts page is the canonical surface for currently-firing alerts."
f.syncLoopState()
f.Lifecycle = append(f.Lifecycle, FindingLifecycleEvent{
At: now,
Type: "auto_resolved",
Message: f.ResolveReason,
From: prevLoopState,
To: f.LoopState,
})
normalizedLoadedState = true
}
// Reset regression counters polluted by bogus auto_resolve
// cycles. Before the category gate (commit b44d5892f) and the
// resource-presence gate that still needs the same treatment,
// findings of event/persistent categories were being
// auto-resolved on absence and re-detected next run, each
// cycle incrementing RegressionCount. The counter became
// unreliable as a signal — the user sees "regressed 6×" but
// most of those regressions weren't real recurrences, they
// were the system fighting itself. Once a finding shows
// evidence of the bogus pattern in its lifecycle, treat the
// entire counter as suspect and reset it; genuine recurrences
// will accrue cleanly from here. Idempotent: after reset,
// the lifecycle is annotated with a regression_counter_reset
// event so the migration only fires once per finding.
if f.ResolvedAt == nil && findingHasBogusAutoResolveCycle(f) {
f.RegressionCount = 0
f.LastRegressionAt = nil
f.Lifecycle = append(f.Lifecycle, FindingLifecycleEvent{
At: time.Now(),
Type: "regression_counter_reset",
Message: "Regression counter reset on migration: prior auto_resolve cycles were driven by the now-removed absence-based auto-resolve paths.",
})
normalizedLoadedState = true
}
// Ensure derived fields are consistent after load.
f.syncLoopState()
s.findings[id] = f
s.byResource[f.ResourceID] = append(s.byResource[f.ResourceID], id)
if f.IsActive() {
s.activeCounts[f.Severity]++
}
}
s.mu.Unlock()
if normalizedLoadedState {
s.scheduleSave()
}
}
return nil
}
// findingHasBogusAutoResolveCycle reports whether the finding's
// RegressionCount was inflated by an unreliable auto_resolve cycle.
//
// For event/persistent categories (reliability, backup, security,
// general) there is no legitimate absence-driven resolution path — the
// underlying condition is a discrete event or persistent state that
// only clears on manual operator action or on a verified fix
// (lifecycle event type "verification_passed"). Any `auto_resolved`
// event on such a finding came from either:
//
// - One of the two now-gated absence paths (message
// "No longer detected by patrol" or "Resource no longer exists
// in infrastructure"), or
// - An LLM `patrol_resolve_finding` tool call (empty message,
// produced by Resolve(_, true)), which the LLM has repeatedly
// misjudged for findings backed by still-active conditions.
//
// In every observed instance, those auto_resolutions were reverted by
// a regressed event within hours when the underlying condition was
// re-detected — the system fighting itself. So once the lifecycle
// shows any auto_resolved on a non-eligible category and the finding
// is currently active again with a regression count, the counter is
// no longer a trustworthy signal and should be reset.
//
// For performance/capacity findings (eligible-category) we don't
// touch the counter: those categories have a legitimate
// metric-cleared resolution path and the absence model is sound.
func findingHasBogusAutoResolveCycle(f *Finding) bool {
if f == nil {
return false
}
if f.RegressionCount == 0 {
return false
}
if CategorySupportsStaleAutoResolve(f.Category) {
return false
}
hasAutoResolve := false
for _, e := range f.Lifecycle {
if e.Type == "regression_counter_reset" {
// Already migrated — must not be re-applied. Genuine
// regressions that accrued after the reset stand on their
// own and the counter is now trustworthy.
return false
}
if e.Type == "auto_resolved" {
hasAutoResolve = true
}
}
return hasAutoResolve
}
// legacyAlertMirrorShape reports whether the finding looks like an
// "Active alert detected" finding produced by the now-removed
// detectAlertSignals → SignalActiveAlert deterministic emitter. The title
// is the only surface that ever produced exactly this string, so matching
// on it is safe even without source/category checks; the additional
// checks tighten the match to avoid touching a hypothetical operator-
// authored finding with the same name. The predicate is state-agnostic:
// callers branch on ResolvedAt to choose between active-retirement and
// resolved-purge behavior.
func legacyAlertMirrorShape(f *Finding) bool {
if f == nil {
return false
}
if f.Title != "Active alert detected" {
return false
}
if f.Source != "ai-analysis" {
return false
}
if f.Category != FindingCategoryGeneral {
return false
}
return true
}
// SweepWillFixLaterReminders proactively wakes will_fix_later commitments
// whose RemindAt deadline has passed without waiting for re-detection. The
// existing wake path in Add() only fires on re-detection — if patrol's
// detection is sparse or the operational issue transiently clears between
// runs, an overdue commitment can sit forever without surfacing. The sweep
// closes that gap by walking all dismissed-will_fix_later findings on a
// timer and converging them onto the same in-memory state the re-detection
// path would have produced.
//
// Mirrors the field-clearing of the will_fix_later branch in Add (around
// findings.go line 1366): clear DismissedReason, Suppressed, RemindAt,
// AcknowledgedAt; keep UserNote because the operator's promise is still
// relevant context; bump RemindCount; emit a "reminded" lifecycle event.
//
// Returns the number of findings woken. Caller is patrol_run.remindSweepLoop.
func (s *FindingsStore) SweepWillFixLaterReminders(now time.Time) int {
s.mu.Lock()
swept := 0
for _, f := range s.findings {
if f == nil {
continue
}
if f.DismissedReason != DismissReasonWillFixLater {
continue
}
if f.RemindAt == nil {
continue
}
if !now.After(*f.RemindAt) {
continue
}
meta := map[string]string{
"reason_was": f.DismissedReason,
"remind_at": f.RemindAt.Format(time.RFC3339),
"trigger": "sweep",
}
f.RemindCount++
s.appendLifecycleLocked(f, "reminded", "will_fix_later remind-at deadline passed (sweep); re-surfacing finding", string(FindingLoopStateDismissed), string(FindingLoopStateDetected), meta)
f.DismissedReason = ""
f.Suppressed = false
f.RemindAt = nil
f.AcknowledgedAt = nil
f.LoopState = string(FindingLoopStateDetected)
swept++
}
s.mu.Unlock()
if swept > 0 {
s.scheduleSave()
}
return swept
}
// scheduleSave schedules a debounced save operation
// This method is lock-safe and can be called without holding the store lock.
func (s *FindingsStore) scheduleSave() {
s.mu.Lock()
if s.persistence == nil || s.savePending {
s.mu.Unlock()
return
}
s.savePending = true
saveDebounce := s.saveDebounce
s.saveTimer = time.AfterFunc(saveDebounce, func() {
s.mu.Lock()
s.savePending = false
// Make a copy for saving, excluding demo findings
findingsCopy := make(map[string]*Finding, len(s.findings))
for id, f := range s.findings {
// Skip demo findings - they should never be persisted
if strings.HasPrefix(id, "demo-") {
continue
}
copy := *f
findingsCopy[id] = ©
}
rulesCopy := make(map[string]*SuppressionRule, len(s.suppressionRules))
for id, r := range s.suppressionRules {
if r == nil {
continue
}
copy := *r
rulesCopy[id] = ©
}
persistence := s.persistence
onError := s.onSaveError
s.mu.Unlock()
if persistence != nil {
var err error
if p2, ok := persistence.(FindingsPersistenceWithSuppression); ok {
err = p2.SaveFindingsAndSuppression(findingsCopy, rulesCopy)
} else {
err = persistence.SaveFindings(findingsCopy)
}
if err != nil {
// Track the error for visibility
s.mu.Lock()
s.lastSaveError = err
s.mu.Unlock()
// Call error callback if set
if onError != nil {
onError(err)
}
} else {
// Clear error and update timestamp on success
s.mu.Lock()
s.lastSaveError = nil
s.lastSaveTime = time.Now()
s.mu.Unlock()
}
}
})
s.mu.Unlock()
}
// ForceSave immediately saves findings (useful for shutdown)
func (s *FindingsStore) ForceSave() error {
s.mu.Lock()
if s.saveTimer != nil {
s.saveTimer.Stop()
}
s.savePending = false
findingsCopy := make(map[string]*Finding, len(s.findings))
for id, f := range s.findings {
// Skip demo findings - they should never be persisted
if strings.HasPrefix(id, "demo-") {
continue
}
copy := *f
findingsCopy[id] = ©
}
rulesCopy := make(map[string]*SuppressionRule, len(s.suppressionRules))
for id, r := range s.suppressionRules {
if r == nil {
continue
}
copy := *r
rulesCopy[id] = ©
}
persistence := s.persistence
s.mu.Unlock()
if persistence != nil {
if p2, ok := persistence.(FindingsPersistenceWithSuppression); ok {
return p2.SaveFindingsAndSuppression(findingsCopy, rulesCopy)
}
return persistence.SaveFindings(findingsCopy)
}
return nil
}
// SetOnSaveError sets a callback function that will be called when a save operation fails.
// This allows external code (e.g., logging) to be notified of persistence errors.
func (s *FindingsStore) SetOnSaveError(callback func(err error)) {
s.mu.Lock()
defer s.mu.Unlock()
s.onSaveError = callback
}
// GetPersistenceStatus returns the current persistence state:
// - lastError: the most recent save error, or nil if last save succeeded
// - lastSaveTime: when findings were last successfully saved
// - hasPersistence: whether a persistence layer is configured
func (s *FindingsStore) GetPersistenceStatus() (lastError error, lastSaveTime time.Time, hasPersistence bool) {
s.mu.RLock()
defer s.mu.RUnlock()
return s.lastSaveError, s.lastSaveTime, s.persistence != nil
}
// appendLifecycleLocked appends a lifecycle event to a finding and bounds the history.
// Caller must hold s.mu.
func (s *FindingsStore) appendLifecycleLocked(f *Finding, typ, msg, from, to string, meta map[string]string) {
if f == nil {
return
}
e := FindingLifecycleEvent{
At: time.Now(),
Type: typ,
Message: msg,
From: from,
To: to,
Metadata: meta,
}
f.Lifecycle = append(f.Lifecycle, e)
if len(f.Lifecycle) > maxFindingLifecycleEvents {
f.Lifecycle = f.Lifecycle[len(f.Lifecycle)-maxFindingLifecycleEvents:]
}
}
// recordTransitionLifecycleLocked records an open/resolved transition for f.
// Below the flapping threshold it appends the ordinary lifecycle row. Once
// the storm throttler reports the finding is flapping, it stamps f.Flapping
// and folds the churn into one "flapping" row that carries the transition
// count, updating that row in place on every further transition instead of
// appending another regressed/resolved entry. The individual rows recorded
// before the label applied stay in the log as forensic detail.
func (s *FindingsStore) recordTransitionLifecycleLocked(f *Finding, typ, msg, from, to string, meta map[string]string) {
if f == nil {
return
}
var flapping *FindingFlapping
if t := s.stormThrottler; t != nil {
flapping = t.observeFlapLocked(f, time.Now())
}
f.Flapping = flapping
if flapping == nil {
s.appendLifecycleLocked(f, typ, msg, from, to, meta)
return
}
collapsed := map[string]string{
"transition_count": strconv.Itoa(flapping.TransitionCount),
"window_hours": strconv.Itoa(flapping.WindowHours),
"first_transition_at": flapping.FirstTransitionAt.UTC().Format(time.RFC3339),
"last_transition": typ,
}
message := fmt.Sprintf(
"State changed %d times in the last %d hours; individual transitions are collapsed while the finding is flapping",
flapping.TransitionCount,
flapping.WindowHours,
)
if n := len(f.Lifecycle); n > 0 && f.Lifecycle[n-1].Type == FindingLifecycleFlapping {
last := &f.Lifecycle[n-1]
last.At = time.Now()
last.Message = message
last.From = from
last.To = to
last.Metadata = collapsed
return
}
s.appendLifecycleLocked(f, FindingLifecycleFlapping, message, from, to, collapsed)
}
func cloneFindingFlapping(value *FindingFlapping) *FindingFlapping {
if value == nil {
return nil
}
clone := *value
return &clone
}
// findingAlertMirror names the active alert a finding restates.
type findingAlertMirror struct {
AlertID string
AlertType string
}
// StampAlertMirrors records, for every unresolved finding, which active
// alert it mirrors (same resource, same condition). Findings absent from
// the map lose any previous stamp, so a resolved alert releases its
// finding back to the normal list on the next reconcile. The stamp is a
// derived read-model annotation and does not schedule a save.
func (s *FindingsStore) StampAlertMirrors(mirrors map[string]findingAlertMirror) int {
s.mu.Lock()
defer s.mu.Unlock()
changed := 0
for id, f := range s.findings {
mirror, ok := mirrors[id]
if !ok || f.ResolvedAt != nil {
if f.MirrorsAlertID != "" || f.MirrorsAlertType != "" {
f.MirrorsAlertID = ""
f.MirrorsAlertType = ""
changed++
}
continue
}
if f.MirrorsAlertID != mirror.AlertID || f.MirrorsAlertType != mirror.AlertType {
f.MirrorsAlertID = mirror.AlertID
f.MirrorsAlertType = mirror.AlertType
changed++
}
}
return changed
}
func isKnownLoopState(v string) bool {
switch FindingLoopState(v) {
case FindingLoopStateDetected, FindingLoopStateInvestigating, FindingLoopStateRemediationPlanned, FindingLoopStateRemediating, FindingLoopStateRemediationFailed, FindingLoopStateNeedsAttention, FindingLoopStateTimedOut, FindingLoopStateResolved, FindingLoopStateDismissed, FindingLoopStateSnoozed, FindingLoopStateSuppressed:
return true
default:
return false
}
}
func isAllowedLoopTransition(from, to string) bool {
if from == "" || from == to {
return true
}
if !isKnownLoopState(from) || !isKnownLoopState(to) {
return false
}
switch FindingLoopState(from) {
case FindingLoopStateResolved:
// Resolved findings should only remain resolved or reopen as detected.
return to == string(FindingLoopStateResolved) || to == string(FindingLoopStateDetected)
case FindingLoopStateSuppressed:
// Suppressed findings should remain suppressed unless explicitly unsuppressed.
return to == string(FindingLoopStateSuppressed) || to == string(FindingLoopStateDetected)
case FindingLoopStateDismissed:
// Dismissed findings can be re-opened or further suppressed/resolved.
return to == string(FindingLoopStateDismissed) ||
to == string(FindingLoopStateDetected) ||
to == string(FindingLoopStateSuppressed) ||
to == string(FindingLoopStateResolved)
case FindingLoopStateSnoozed:
// Snoozed findings can wake up (detected) or be resolved/suppressed/dismissed.
return to == string(FindingLoopStateSnoozed) ||
to == string(FindingLoopStateDetected) ||
to == string(FindingLoopStateResolved) ||
to == string(FindingLoopStateSuppressed) ||
to == string(FindingLoopStateDismissed)
default:
return true
}
}
func normalizeLoadedFinding(f *Finding) bool {
if f == nil {
return false
}
if f.ResolvedAt == nil && f.LastRegressionAt != nil && f.AcknowledgedAt != nil && !f.AcknowledgedAt.After(*f.LastRegressionAt) {
f.AcknowledgedAt = nil
return true
}
return false
}
func findingSeverityRank(severity FindingSeverity) int {
switch severity {
case FindingSeverityCritical:
return 3
case FindingSeverityWarning:
return 2
case FindingSeverityWatch:
return 1
case FindingSeverityInfo:
return 0
default:
return 0
}
}
func sameFindingResource(a, b *Finding) bool {
if a == nil || b == nil {
return false
}
candidatesA := []string{
strings.TrimSpace(a.ResourceID),
strings.TrimSpace(a.ResourceName),
}
candidatesB := []string{
strings.TrimSpace(b.ResourceID),
strings.TrimSpace(b.ResourceName),
}
for _, left := range candidatesA {
if left == "" {
continue
}
for _, right := range candidatesB {
if right != "" && left == right {
return true
}
}
}
return false
}
func normalizedStorageFindingIdentity(value string) string {
value = strings.TrimSpace(strings.ToLower(value))
if value == "" {
return ""
}
var b strings.Builder
word := ""
flush := func() {
if word == "" {
return
}
switch word {
case "storage", "pool", "array":
word = ""
return
}
if b.Len() > 0 {
b.WriteByte(' ')
}
b.WriteString(word)
word = ""
}
for _, r := range value {
if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') {
word += string(r)
continue
}
flush()
}
flush()
return b.String()
}
func storageFindingIdentityCandidates(f *Finding) []string {
if f == nil {
return nil
}
seen := make(map[string]bool)
raw := []string{f.ResourceID, f.ResourceName}
candidates := make([]string, 0, len(raw))
for _, value := range raw {
normalized := normalizedStorageFindingIdentity(value)
if normalized == "" || seen[normalized] {
continue
}
seen[normalized] = true
candidates = append(candidates, normalized)
}
return candidates
}
func sameStorageFindingIdentity(a, b *Finding) bool {
if sameFindingResource(a, b) {
return true
}
aCandidates := storageFindingIdentityCandidates(a)
bCandidates := storageFindingIdentityCandidates(b)
for _, left := range aCandidates {
for _, right := range bCandidates {
if left == right {
return true
}
}
}
return false
}
func findingTextContainsAny(f *Finding, needles ...string) bool {
if f == nil {
return false
}
text := strings.ToLower(strings.Join([]string{
f.Key,
f.ResourceID,
f.ResourceName,
f.ResourceType,
string(f.Category),
f.Title,
f.Description,
f.Impact,
f.Recommendation,
f.Evidence,
}, " "))
for _, needle := range needles {
if strings.Contains(text, needle) {
return true
}
}
return false
}
func storageFindingLooksLikeStorageRisk(f *Finding) bool {
if f == nil {
return false
}
if canonicalFindingResourceType(f.ResourceType) == "storage" {
return true
}
return findingTextContainsAny(f, "storage", "pool", "array", "unraid", "zfs", "ceph")
}
func storageFindingHasCapacitySignal(f *Finding) bool {
if f == nil {
return false
}
if f.Category == FindingCategoryCapacity {
return true
}
return findingTextContainsAny(f, "capacity", "usage", "used", "free", "full", "disk", "%", "tb")
}
func equivalentStorageCapacityFinding(existing, incoming *Finding) bool {
if !sameStorageFindingIdentity(existing, incoming) {
return false
}
if !storageFindingLooksLikeStorageRisk(existing) || !storageFindingLooksLikeStorageRisk(incoming) {
return false
}
if !storageFindingHasCapacitySignal(existing) || !storageFindingHasCapacitySignal(incoming) {
return false
}
return existing.Category == FindingCategoryCapacity || incoming.Category == FindingCategoryCapacity
}
func normalizedFindingTitle(title string) string {
return normalizeFindingKey(title)
}
func equivalentActiveFinding(existing, incoming *Finding) bool {
if existing == nil || incoming == nil || existing.ID == incoming.ID {
return false
}
sameResource := sameFindingResource(existing, incoming)
sameStorageIdentity := !sameResource && sameStorageFindingIdentity(existing, incoming)
if !existing.IsActive() || (!sameResource && !sameStorageIdentity) {
return false
}
existingTitle := normalizedFindingTitle(existing.Title)
incomingTitle := normalizedFindingTitle(incoming.Title)
if existingTitle != "" && existingTitle == incomingTitle {
return true
}
if equivalentStorageCapacityFinding(existing, incoming) {
return true
}
if existing.Category != "" && incoming.Category != "" && existing.Category != incoming.Category {
return false
}
return SemanticSimilarity(existing, incoming) >= 0.78
}
func (s *FindingsStore) findEquivalentActiveFindingLocked(incoming *Finding) *Finding {
if incoming == nil {
return nil
}
var best *Finding
bestScore := 0.0
for _, existing := range s.findings {
if !equivalentActiveFinding(existing, incoming) {
continue
}
score := SemanticSimilarity(existing, incoming)
if best == nil || score > bestScore {
best = existing
bestScore = score
}
}
return best
}
func findingHasDuplicateMergeEvent(f *Finding, duplicateID string) bool {
if f == nil || duplicateID == "" {
return false
}
for _, event := range f.Lifecycle {
if event.Type != "duplicate_merged" {
continue
}
if event.Metadata != nil && event.Metadata["duplicate_id"] == duplicateID {
return true
}
}
return false
}
func appendFindingResourceIndexUnique(index map[string][]string, resourceID, findingID string) {
resourceID = strings.TrimSpace(resourceID)
findingID = strings.TrimSpace(findingID)
if resourceID == "" || findingID == "" {
return
}
for _, existingID := range index[resourceID] {
if existingID == findingID {
return
}
}
index[resourceID] = append(index[resourceID], findingID)
}
func (s *FindingsStore) mergeEquivalentFindingLocked(existing, incoming *Finding) {
now := time.Now()
previousSeverity := existing.Severity
incomingSeverityRank := findingSeverityRank(incoming.Severity)
existingSeverityRank := findingSeverityRank(existing.Severity)
existing.LastSeenAt = now
existing.TimesRaised++
if incomingSeverityRank >= existingSeverityRank {
existing.Severity = incoming.Severity
if incoming.Title != "" {
existing.Title = incoming.Title
}
if incoming.Description != "" {
existing.Description = incoming.Description
}
if incoming.Impact != "" {
existing.Impact = incoming.Impact
}
if incoming.Recommendation != "" {
existing.Recommendation = incoming.Recommendation
}
if incoming.Evidence != "" {
existing.Evidence = incoming.Evidence
}
}
if existing.Key == "" && incoming.Key != "" {
existing.Key = incoming.Key
}
if existing.AlertIdentifier == "" && incoming.AlertIdentifier != "" {
existing.AlertIdentifier = incoming.AlertIdentifier
}
if existing.Node == "" && incoming.Node != "" {
existing.Node = incoming.Node
}
if criticality := normalizeResourceCriticality(incoming.ResourceCriticality); criticality != "" {
existing.ResourceCriticality = criticality
}
s.syncLoopStateLocked(existing)
if incoming.ResourceID != "" && incoming.ResourceID != existing.ResourceID {
appendFindingResourceIndexUnique(s.byResource, incoming.ResourceID, existing.ID)
}
if !findingHasDuplicateMergeEvent(existing, incoming.ID) {
s.appendLifecycleLocked(existing, "duplicate_merged", "Merged an equivalent active Patrol finding into this issue", existing.LoopState, existing.LoopState, map[string]string{
"duplicate_id": incoming.ID,
"duplicate_key": incoming.Key,
"duplicate_severity": string(incoming.Severity),
})
}
if previousSeverity != existing.Severity && existing.IsActive() {
if s.activeCounts[previousSeverity] > 0 {
s.activeCounts[previousSeverity]--
}
s.activeCounts[existing.Severity]++
}
}
// findOperatorStateDismissCause returns the most recent
// `operator_state_cause` metadata value from the finding's lifecycle, or
// "" if the finding has not been auto-dismissed by operator-state
// suppression. Used by the dismissed-branch wake logic in Add to
// distinguish operator-state-driven dismissals (which should wake when
// suppression lifts) from manual operator dismissals (which stay quiet
// until severity escalates).
func findOperatorStateDismissCause(f *Finding) string {
if f == nil {
return ""
}
for i := len(f.Lifecycle) - 1; i >= 0; i-- {
ev := f.Lifecycle[i]
if ev.Type != "dismissed" {
continue
}
if cause, ok := ev.Metadata["operator_state_cause"]; ok && cause != "" {
return cause
}
// First "dismissed" event without operator_state_cause is a
// manual operator dismissal that supersedes any earlier
// auto-dismiss; stop scanning so a stale earlier cause does
// not falsely trigger a wake.
return ""
}
return ""
}
// syncLoopStateLocked recomputes loop state and records a lifecycle event if it changed.
// Caller must hold s.mu.
func (s *FindingsStore) syncLoopStateLocked(f *Finding) {
if f == nil {
return
}
prev := f.LoopState
f.syncLoopState()
if prev == f.LoopState {
return
}
if !isAllowedLoopTransition(prev, f.LoopState) {
next := f.LoopState
f.LoopState = prev
s.appendLifecycleLocked(f, "loop_transition_violation", "Blocked invalid loop-state transition", prev, next, map[string]string{
"from": prev,
"to": next,
})
return
}
// Successful loop-state transitions are not recorded as a generic
// "loop_state" lifecycle event. Every caller of syncLoopStateLocked
// already emits a semantic event for the transition it caused
// ("auto_resolved", "regressed", "dismissed", "acknowledged",
// "snoozed", etc.) with the same from/to. Emitting an additional
// "loop_state" event here just duplicated every row in the finding's
// lifecycle drawer. The loop_transition_violation branch above stays:
// it's the only signal that a transition was rejected, not a duplicate.
}
// operatorResourceIDLocked resolves Add's operator-state lookup identity.
// Caller holds s.mu; an update without an identity uses the retained finding.
func (s *FindingsStore) operatorResourceIDLocked(f *Finding) string {
if id := strings.TrimSpace(f.ResourceID); id != "" {
return id
}
if existing := s.findings[f.ID]; existing != nil {
return strings.TrimSpace(existing.ResourceID)
}
return ""
}
// Add adds or updates a finding
// If a finding with the same ID exists, it updates LastSeenAt and increments TimesRaised
// If the finding is suppressed or dismissed, it may be skipped
// Returns true if this is a new finding
func (s *FindingsStore) Add(f *Finding) bool {
s.mu.Lock()
// Operator-state adapters can wait on storage or read this store. Do not
// retain the findings lock across that external call: shutdown needs it to
// persist findings already accepted, even if a new lookup is stalled.
var operatorProjection ResourceOperatorStateProjection
operatorProjectionOK := false
operatorStateProviderPresent := false
for {
provider := s.resourceOperatorStateProvider
resourceID := s.operatorResourceIDLocked(f)
if provider == nil || resourceID == "" {
break
}
revision := s.resourceOperatorStateProviderRevision
s.mu.Unlock()
projection, ok := provider.OperatorStateProjection(resourceID, time.Now())
s.mu.Lock()
// A replaced provider or rebound fallback identity invalidates the
// result. Re-read before committing suppression or criticality.
if revision != s.resourceOperatorStateProviderRevision || resourceID != s.operatorResourceIDLocked(f) {
continue
}
operatorProjection, operatorProjectionOK = projection, ok
operatorStateProviderPresent = true
break
}
if f.ResourceType == "" {
f.ResourceType = inferFindingResourceType(f.ResourceID, f.ResourceName)
}
// Keep loop state derived from the current flags/outcomes.
f.syncLoopState()
existing, exists := s.findings[f.ID]
if operatorStateProviderPresent {
if operatorProjectionOK {
f.ResourceCriticality = normalizeResourceCriticality(operatorProjection.Criticality)
} else {
f.ResourceCriticality = ""
}
} else if s.resourceOperatorStateProvider == nil {
f.ResourceCriticality = normalizeResourceCriticality(f.ResourceCriticality)
}
if exists {
wasResolved := existing.ResolvedAt != nil
if operatorStateProviderPresent || f.ResourceCriticality != "" {
existing.ResourceCriticality = normalizeResourceCriticality(f.ResourceCriticality)
}
if existing.ResourceType == "" {
if f.ResourceType != "" {
existing.ResourceType = f.ResourceType
} else {
existing.ResourceType = inferFindingResourceType(existing.ResourceID, existing.ResourceName)
}
}
// Check if dismissed or suppressed - only update if severity has escalated,
// or if a "will_fix_later" remind-at deadline has passed, or if the
// operator-state suppression that auto-dismissed the finding is no
// longer active.
if existing.DismissedReason != "" || existing.Suppressed {
severityOrder := map[FindingSeverity]int{
FindingSeverityInfo: 0,
FindingSeverityWatch: 1,
FindingSeverityWarning: 2,
FindingSeverityCritical: 3,
}
// "will_fix_later" with a passed RemindAt is a real operator
// commitment that has expired — surface the finding again. This
// is the only case where a same-or-lower-severity re-detection
// reactivates a dismissed finding; "expected_behavior" stays
// quiet, "not_an_issue" stays suppressed.
willFixReminderDue := existing.DismissedReason == DismissReasonWillFixLater &&
existing.RemindAt != nil &&
time.Now().After(*existing.RemindAt) &&
!existing.Suppressed
// Operator-state-driven auto-dismiss should be reversible when the
// underlying suppression ends. A maintenance window has a defined
// end; an IntentionallyOffline flag can be cleared. Findings
// auto-dismissed under either signal must wake at re-detection
// once the suppression no longer applies, otherwise the
// time-bounded suppression silently becomes permanent.
operatorStateLifted := false
if !willFixReminderDue && existing.DismissedReason == "expected_behavior" && !existing.Suppressed {
if cause := findOperatorStateDismissCause(existing); cause != "" {
if operatorStateProviderPresent {
currentlySuppressing := operatorProjectionOK && operatorProjection.suppressesFinding(f)
operatorStateLifted = !currentlySuppressing
} else {
// No provider currently wired but the finding was
// previously auto-dismissed by one — treat as lifted
// rather than keeping the suppression alive forever.
operatorStateLifted = true
}
}
}
if !willFixReminderDue && !operatorStateLifted &&
severityOrder[f.Severity] <= severityOrder[existing.Severity] {
// New severity is same or lower and no wake condition fired — don't reactivate.
existing.LastSeenAt = time.Now()
existing.TimesRaised++
s.appendLifecycleLocked(existing, "seen_while_suppressed", "Re-detected while dismissed/suppressed with non-escalated severity", existing.LoopState, existing.LoopState, nil)
s.mu.Unlock()
s.scheduleSave()
return false
}
if willFixReminderDue {
prevRemindAt := existing.RemindAt
meta := map[string]string{"reason_was": existing.DismissedReason}
if prevRemindAt != nil {
meta["remind_at"] = prevRemindAt.Format(time.RFC3339)
}
existing.RemindCount++
s.appendLifecycleLocked(existing, "reminded", "will_fix_later remind-at deadline passed; re-surfacing finding", string(FindingLoopStateDismissed), string(FindingLoopStateDetected), meta)
}
if operatorStateLifted {
meta := map[string]string{}
if cause := findOperatorStateDismissCause(existing); cause != "" {
meta["previous_cause"] = cause
}
s.appendLifecycleLocked(existing, "suppression_lifted", "Operator-state suppression no longer applies; re-surfacing finding", string(FindingLoopStateDismissed), string(FindingLoopStateDetected), meta)
}
// Severity escalated, remind-at fired, or operator-state lifted — clear dismissal and reactivate.
existing.DismissedReason = ""
existing.Suppressed = false
existing.RemindAt = nil
if !willFixReminderDue {
// On severity escalation or operator-state-lift we drop the
// operator's note (the suppression that justified it is gone);
// on a remind-at wake we keep it because the operator's
// promise is still relevant context.
existing.UserNote = ""
}
existing.AcknowledgedAt = nil
}
// Same-key re-detections normally describe the same issue, but the
// LLM-assigned key can collide: a substantially different report
// arriving under an existing finding's identity overwrites its text
// below, silently absorbing a distinct issue into this finding's
// history (and, on a resolved finding, counting it as a regression
// of the old issue). Key forking would be worse — LLM rephrasings
// would split one real issue into duplicate findings — so the merge
// stands, and the identity shift is recorded honestly in the
// lifecycle with the previous title preserved. Detection compares
// title keyword overlap: resource/category/key are equal by
// construction, so the text is the only discriminating signal, and
// plain rephrasings of the same issue share core keywords.
if existing.Title != "" && f.Title != "" && existing.Title != f.Title &&
keywordOverlap(existing.Title, f.Title) <= findingIdentityShiftMaxTitleOverlap {
log.Info().
Str("finding_id", existing.ID).
Str("key", existing.Key).
Str("previous_title", existing.Title).
Str("new_title", f.Title).
Msg("Finding key collision: re-detection replaced this finding's content with a substantially different report")
s.appendLifecycleLocked(existing, "content_replaced",
"Re-detected with substantially different details; the previous report's title is preserved in this event",
existing.LoopState, existing.LoopState,
map[string]string{"previous_title": existing.Title, "new_title": f.Title})
}
// Update existing finding
existing.LastSeenAt = time.Now()
existing.Description = f.Description
existing.Impact = f.Impact
existing.Recommendation = f.Recommendation
existing.Evidence = f.Evidence
// The runtime failure finding is re-raised with a fresh cause each run
// (provider fault, then budget exhaustion); the cause must follow the
// prose it explains, or the Patrol page routes the operator to the
// wrong fix after a restart clears the in-memory block state (#1789).
existing.FailureCause = f.FailureCause
if f.ObjectiveContext != nil {
existing.ObjectiveContext = clonePatrolObjectiveContext(f.ObjectiveContext)
}
existing.Title = f.Title // Update title in case LLM phrased it better
existing.Severity = f.Severity
existing.TimesRaised++ // Track recurrence
if wasResolved {
prevResolvedAt := existing.ResolvedAt
prevResolveReason := existing.ResolveReason
hadAcknowledgement := existing.AcknowledgedAt != nil
// Capture the prior successful fix as operational memory before
// the investigation record is cleared. This preserves "what worked
// last time" across regressions so the next investigation does
// not start from blank context.
if existing.InvestigationRecord != nil && existing.InvestigationRecord.ProposedFix != nil {
if desc := strings.TrimSpace(existing.InvestigationRecord.ProposedFix.Description); desc != "" {
existing.PreviousResolvedFixSummary = desc
}
}
existing.ResolvedAt = nil
existing.AutoResolved = false
existing.ResolveReason = ""
existing.AcknowledgedAt = nil
// Reset investigation loop metadata when a previously resolved issue reappears.
existing.InvestigationSessionID = ""
existing.InvestigationStatus = ""
existing.InvestigationOutcome = ""
existing.LastInvestigatedAt = nil
existing.InvestigationAttempts = 0
existing.InvestigationRecord = nil
existing.RegressionCount++
now := time.Now()
existing.LastRegressionAt = &now
s.occurrenceStarts[existing.ID] = now
meta := map[string]string{}
if prevResolvedAt != nil {
meta["previous_resolved_at"] = prevResolvedAt.Format(time.RFC3339)
}
if prevResolveReason != "" {
meta["previous_resolve_reason"] = prevResolveReason
}
if hadAcknowledgement {
meta["previous_acknowledged"] = "true"
}
s.recordTransitionLifecycleLocked(existing, "regressed", "Finding re-detected after resolution", string(FindingLoopStateResolved), string(FindingLoopStateDetected), meta)
if existing.IsActive() {
s.activeCounts[existing.Severity]++
}
}
s.syncLoopStateLocked(existing)
// Re-detections of an existing finding are heartbeats, not transitions.
// TimesRaised and LastSeenAt already track recurrence. The actual
// transition events ("regressed", "reminded", "suppression_lifted",
// etc.) are emitted from their own branches above; appending an
// additional "detected (same -> same)" event on every Patrol scan
// pollutes the lifecycle with no-op rows.
severity := existing.Severity
s.mu.Unlock()
s.scheduleSave()
// Bypass debounce for warning+ findings to avoid data loss on crash
if severity == FindingSeverityWarning || severity == FindingSeverityCritical {
_ = s.ForceSave()
}
return false
}
// New finding - check if resource+category is suppressed
if s.isSuppressedInternal(f.ResourceID, f.Category) {
s.mu.Unlock()
return false
}
if equivalent := s.findEquivalentActiveFindingLocked(f); equivalent != nil {
s.mergeEquivalentFindingLocked(equivalent, f)
severity := equivalent.Severity
s.mu.Unlock()
s.scheduleSave()
if severity == FindingSeverityWarning || severity == FindingSeverityCritical {
_ = s.ForceSave()
}
return false
}
// New finding
if f.DetectedAt.IsZero() {
f.DetectedAt = time.Now()
}
f.LastSeenAt = time.Now()
if f.TimesRaised <= 0 {
f.TimesRaised = 1
}
s.syncLoopStateLocked(f)
s.appendLifecycleLocked(f, "detected", "Detected by Pulse Patrol", "", f.LoopState, nil)
// Operator-state suppression: when the operator has set per-resource
// intent that suppresses findings, auto-dismiss the finding as
// expected_behavior with a reason that names the cause. Maintenance
// windows take priority because they're time-bounded ("expect this
// noise until the window closes"); monitoring and lifecycle policy is
// the indefinite fallback, scoped per finding by
// monitoringSuppressionCauseFor (expected-offline silences only
// availability findings). The finding still lands in durable history so
// the operator can audit what tripped during the suppression;
// severity escalation paths still wake it later if the underlying
// condition stays after the operator clears the state.
if operatorStateProviderPresent {
if projection, ok := operatorProjection, operatorProjectionOK; ok {
now := time.Now()
switch {
case projection.MaintenanceWindow != nil:
window := projection.MaintenanceWindow
f.DismissedReason = "expected_behavior"
f.AcknowledgedAt = &now
note := fmt.Sprintf("Resource is in operator-set maintenance window until %s.", window.EndAt.Format(time.RFC3339))
if window.Inherited {
note = fmt.Sprintf("Resource is covered by an operator-set maintenance window on a parent resource until %s.", window.EndAt.Format(time.RFC3339))
}
noteParts := []string{note}
if reason := strings.TrimSpace(window.Reason); reason != "" {
noteParts = append(noteParts, "Reason: "+reason)
}
f.UserNote = strings.Join(noteParts, " ")
metadata := map[string]string{
"reason": "expected_behavior",
"operator_state_cause": "maintenance_window",
"maintenance_end_at": window.EndAt.Format(time.RFC3339),
}
if window.Inherited {
metadata["maintenance_inherited"] = "true"
metadata["maintenance_source_id"] = window.SourceID
}
s.appendLifecycleLocked(f, "dismissed", "Auto-acknowledged: resource in maintenance window", f.LoopState, string(FindingLoopStateDismissed), metadata)
s.syncLoopStateLocked(f)
case projection.monitoringSuppressionCauseFor(f) != "":
cause := projection.monitoringSuppressionCauseFor(f)
f.DismissedReason = "expected_behavior"
f.AcknowledgedAt = &now
switch cause {
case "intentionally_offline":
f.UserNote = "Resource is marked intentionally offline by the operator, so Patrol keeps its availability findings quiet. Clear the flag on the resource detail surface to resume them."
case "expected_offline":
f.UserNote = "Resource is marked expected offline, so Patrol keeps its availability findings quiet. Set monitoring back to Normal on the resource detail surface to resume them."
default:
f.UserNote = "Resource monitoring policy suppresses Patrol attention. Return monitoring and lifecycle to active defaults on the resource detail surface to resume findings."
}
s.appendLifecycleLocked(f, "dismissed", "Auto-acknowledged: resource monitoring policy suppresses attention", f.LoopState, string(FindingLoopStateDismissed), map[string]string{
"reason": "expected_behavior",
"operator_state_cause": cause,
})
s.syncLoopStateLocked(f)
}
}
}
s.findings[f.ID] = f
s.byResource[f.ResourceID] = append(s.byResource[f.ResourceID], f.ID)
if f.IsActive() {
s.activeCounts[f.Severity]++
}
// Storm throttler hook: still inside the new-finding branch with
// s.mu held so the observer reads a consistent index. The observer
// owns its own internal mutex and must not call back into the
// store; its return is dispatched after s.mu is released below.
var pendingStormFinding *Finding
if t := s.stormThrottler; t != nil {
pendingStormFinding = t.observeLocked(f, time.Now())
}
severity := f.Severity
s.mu.Unlock()
s.scheduleSave()
// Bypass debounce for warning+ findings to avoid data loss on crash
if severity == FindingSeverityWarning || severity == FindingSeverityCritical {
_ = s.ForceSave()
}
if pendingStormFinding != nil {
if pendingStormFinding.ResolvedAt != nil {
s.ResolveWithReason(pendingStormFinding.ID, stormResolveReason)
} else {
s.Add(pendingStormFinding)
}
}
return true
}
// Resolve marks a finding as resolved
func (s *FindingsStore) Resolve(id string, auto bool) bool {
s.mu.Lock()
f, exists := s.findings[id]
if !exists || !f.IsActive() {
s.mu.Unlock()
return false
}
now := time.Now()
f.ResolvedAt = &now
f.AutoResolved = auto
etype := "resolved"
if auto {
etype = "auto_resolved"
}
s.recordTransitionLifecycleLocked(f, etype, f.ResolveReason, f.LoopState, string(FindingLoopStateResolved), nil)
s.syncLoopStateLocked(f)
s.activeCounts[f.Severity]--
s.mu.Unlock()
s.scheduleSave()
return true
}
// ResolveWithReason marks a finding as resolved with a specific reason string.
// This is used by auto-resolution to distinguish why a finding was resolved.
func (s *FindingsStore) ResolveWithReason(id string, reason string) bool {
s.mu.Lock()
f, exists := s.findings[id]
if !exists || !f.IsActive() {
s.mu.Unlock()
return false
}
now := time.Now()
f.ResolvedAt = &now
f.AutoResolved = true
f.ResolveReason = reason
s.recordTransitionLifecycleLocked(f, "auto_resolved", reason, f.LoopState, string(FindingLoopStateResolved), nil)
s.syncLoopStateLocked(f)
s.activeCounts[f.Severity]--
s.mu.Unlock()
s.scheduleSave()
return true
}
// Acknowledge marks a finding as acknowledged
func (s *FindingsStore) Acknowledge(id string) bool {
s.mu.Lock()
f, exists := s.findings[id]
if !exists {
s.mu.Unlock()
return false
}
now := time.Now()
f.AcknowledgedAt = &now
s.appendLifecycleLocked(f, "acknowledged", "", "", "", nil)
s.mu.Unlock()
s.scheduleSave()
return true
}
// Snooze hides a finding for the specified duration
// Common durations: 1h, 24h, 7d (168h)
func (s *FindingsStore) Snooze(id string, duration time.Duration) bool {
s.mu.Lock()
f, exists := s.findings[id]
if !exists || f.IsResolved() {
s.mu.Unlock()
return false
}
// If was previously active (not snoozed), decrement count
if f.SnoozedUntil == nil || time.Now().After(*f.SnoozedUntil) {
s.activeCounts[f.Severity]--
}
until := time.Now().Add(duration)
f.SnoozedUntil = &until
s.appendLifecycleLocked(f, "snoozed", "", f.LoopState, string(FindingLoopStateSnoozed), map[string]string{
"until": until.Format(time.RFC3339),
})
s.syncLoopStateLocked(f)
s.mu.Unlock()
s.scheduleSave()
return true
}
// Unsnooze removes the snooze from a finding, making it active again
func (s *FindingsStore) Unsnooze(id string) bool {
s.mu.Lock()
f, exists := s.findings[id]
if !exists || f.IsResolved() {
s.mu.Unlock()
return false
}
if f.SnoozedUntil != nil {
prev := f.SnoozedUntil
f.SnoozedUntil = nil
s.activeCounts[f.Severity]++
meta := map[string]string{}
if prev != nil {
meta["previous_until"] = prev.Format(time.RFC3339)
}
s.appendLifecycleLocked(f, "unsnoozed", "", string(FindingLoopStateSnoozed), string(FindingLoopStateDetected), meta)
}
s.syncLoopStateLocked(f)
s.mu.Unlock()
s.scheduleSave()
return true
}
// Dismiss marks a finding as dismissed with a reason and optional note
// Reasons: "not_an_issue", "expected_behavior", "will_fix_later"
//
// Behavior by reason:
// - "not_an_issue": Permanent suppression (true false positive in detection logic)
// - "expected_behavior": Acknowledged only (removed from active list, stays in dismissed history)
// - "will_fix_later": Acknowledged only (removed from active list, stays in dismissed history)
//
// Rationale: Only true false positives ("not_an_issue") should be permanently suppressed.
// For "expected_behavior" and "will_fix_later", the finding stays visible (transparent)
// but is marked as acknowledged so the user knows they've reviewed it.
// Severity escalation will still clear the dismissal and reactivate the finding.
func (s *FindingsStore) Dismiss(id, reason, note string) bool {
s.mu.Lock()
f, exists := s.findings[id]
if !exists {
s.mu.Unlock()
return false
}
f.DismissedReason = reason
if note != "" {
f.UserNote = note
}
// Mark as acknowledged for all dismiss reasons
now := time.Now()
f.AcknowledgedAt = &now
dismissMeta := map[string]string{"reason": reason}
// Only "not_an_issue" creates permanent suppression
// This is for true false positives where the detection logic is wrong
if reason == "not_an_issue" {
f.Suppressed = true
f.RemindAt = nil
}
// "will_fix_later" is an operational commitment, not a permanent silence.
// Default a 7-day remind-at so the next re-detection after the window
// surfaces the finding with a "reminded" lifecycle event instead of being
// swallowed forever like "expected_behavior".
if reason == DismissReasonWillFixLater {
remindAt := now.Add(DefaultWillFixLaterRemindAfter)
f.RemindAt = &remindAt
dismissMeta["remind_at"] = remindAt.Format(time.RFC3339)
} else if reason != "" {
f.RemindAt = nil
}
s.appendLifecycleLocked(f, "dismissed", "", f.LoopState, string(FindingLoopStateDismissed), dismissMeta)
s.syncLoopStateLocked(f)
// For "expected_behavior":
// - Finding stays visible (not suppressed, not snoozed)
// - But is marked as dismissed/acknowledged so user knows they've reviewed it
// - Severity escalation will clear DismissedReason and reactivate
// For "will_fix_later":
// - Same as expected_behavior, plus RemindAt is set; the next re-detection
// after RemindAt clears the dismissal and the finding surfaces again
// with a "reminded" lifecycle event.
s.mu.Unlock()
s.scheduleSave()
return true
}
// Undismiss reverts a dismissed finding back to active state
// This clears DismissedReason, Suppressed, and AcknowledgedAt
// allowing the finding to be raised again
func (s *FindingsStore) Undismiss(id string) bool {
s.mu.Lock()
f, exists := s.findings[id]
if !exists {
s.mu.Unlock()
return false
}
// Check if it was actually dismissed
if f.DismissedReason == "" && !f.Suppressed {
s.mu.Unlock()
return false
}
s.undismissLocked(f)
s.mu.Unlock()
s.scheduleSave()
return true
}
func (s *FindingsStore) undismissLocked(f *Finding) {
// Clear dismissal state
f.DismissedReason = ""
f.Suppressed = false
f.AcknowledgedAt = nil
f.RemindAt = nil
s.appendLifecycleLocked(f, "undismissed", "", string(FindingLoopStateDismissed), string(FindingLoopStateDetected), nil)
// Keep UserNote in case user wants to see their notes
// If it was resolved, don't reactivate - user should manually reopen
// But if it's not resolved, it becomes active again
if f.ResolvedAt == nil && !f.IsSnoozed() {
s.activeCounts[f.Severity]++
}
s.syncLoopStateLocked(f)
}
// UpdateInvestigationRecord stores the durable investigation record for a finding.
func (s *FindingsStore) UpdateInvestigationRecord(id string, record *aicontracts.InvestigationRecord) bool {
s.mu.Lock()
f, exists := s.findings[id]
if !exists {
s.mu.Unlock()
return false
}
if record != nil && !f.acceptsInvestigationResult(record.SessionID, record.StartedAt) {
s.mu.Unlock()
return false
}
f.InvestigationRecord = record
meta := map[string]string{}
if record != nil {
meta["record_id"] = record.ID
meta["status"] = string(record.Status)
meta["outcome"] = string(record.Outcome)
}
s.appendLifecycleLocked(f, "investigation_record_updated", "Investigation record updated", f.LoopState, f.LoopState, meta)
s.mu.Unlock()
s.scheduleSave()
return true
}
// SetUserNote updates the user note on a finding
func (s *FindingsStore) SetUserNote(id, note string) bool {
s.mu.Lock()
f, exists := s.findings[id]
if !exists {
s.mu.Unlock()
return false
}
f.UserNote = note
msg := "User note updated"
if strings.TrimSpace(note) == "" {
msg = "User note cleared"
}
s.appendLifecycleLocked(f, "user_note_updated", msg, "", "", nil)
s.mu.Unlock()
s.scheduleSave()
return true
}
// StampCapacityForecasts attaches the given deterministic capacity forecasts
// to matching active findings, joined by resource ID. A forecast applies to a
// finding when the finding concerns capacity on that resource (capacity
// category, or a storage/disk resource). The forecast is recomputed every run,
// so this is per-run data maintenance, not operator history: no lifecycle
// event is recorded and findings already holding an equal forecast are skipped
// to avoid spurious saves. Returns the number of findings updated.
func (s *FindingsStore) StampCapacityForecasts(byResource map[string]CapacityForecast) int {
if s == nil || len(byResource) == 0 {
return 0
}
s.mu.Lock()
changed := 0
for _, f := range s.findings {
if f == nil || !f.IsActive() {
continue
}
fc, ok := byResource[f.ResourceID]
if !ok {
continue
}
if !capacityForecastAppliesTo(f) {
continue
}
if f.CapacityForecast != nil && *f.CapacityForecast == fc {
continue
}
fcCopy := fc
f.CapacityForecast = &fcCopy
changed++
}
s.mu.Unlock()
if changed > 0 {
s.scheduleSave()
}
return changed
}
// capacityForecastAppliesTo reports whether a deterministic capacity forecast
// is meaningful for this finding: a capacity finding, or any finding on a
// storage/disk resource where utilization trend is directly relevant.
func capacityForecastAppliesTo(f *Finding) bool {
if f == nil {
return false
}
if f.Category == FindingCategoryCapacity {
return true
}
switch f.ResourceType {
case "storage", "physical_disk":
return true
default:
return false
}
}
// UpdateInvestigationOutcome updates the investigation outcome on a finding
func (s *FindingsStore) UpdateInvestigationOutcome(id, outcome string) bool {
s.mu.Lock()
f, exists := s.findings[id]
if !exists {
s.mu.Unlock()
return false
}
s.updateInvestigationOutcomeLocked(f, outcome)
s.mu.Unlock()
s.scheduleSave()
return true
}
// ReconcileInvestigationOutcome projects an authoritative session result only
// onto the occurrence that owns it. A finding ID survives regressions, while
// the investigation session and regression boundary identify the current work.
// Check and mutation share the store lock so a concurrent re-detection cannot
// be closed by a result accepted against its previous occurrence.
func (s *FindingsStore) ReconcileInvestigationOutcome(id string, session *aicontracts.InvestigationSession, outcome string) (applied, changed bool) {
if session == nil || session.FindingID != id {
return false, false
}
s.mu.Lock()
f := s.findings[id]
if f == nil || !f.IsCurrentInvestigation(session) {
s.mu.Unlock()
return false, false
}
changed = f.InvestigationOutcome != outcome
if changed {
s.updateInvestigationOutcomeLocked(f, outcome)
}
s.mu.Unlock()
if changed {
s.scheduleSave()
}
return true, changed
}
// IsCurrentInvestigation distinguishes current finding work from historical
// sessions retained under the same finding ID.
func (f *Finding) IsCurrentInvestigation(session *aicontracts.InvestigationSession) bool {
return f != nil && session != nil && session.FindingID == f.ID &&
f.acceptsInvestigationResult(session.SessionID, session.StartedAt)
}
func (f *Finding) acceptsInvestigationResult(sessionID string, startedAt time.Time) bool {
if f.InvestigationSessionID != "" && f.InvestigationSessionID != sessionID {
return false
}
// Re-detection clears the session reference. Its persisted regression time
// still excludes old sessions during the gap before a new one starts, and
// after a process restart. Missing historical time cannot prove ownership.
return f.LastRegressionAt == nil || (!startedAt.IsZero() && !startedAt.Before(*f.LastRegressionAt))
}
// liveRunOccurrenceLocked returns when the finding's current occurrence began,
// as far as a run of this process can tell: the latest regression this process
// saw, or the finding's removal, after which the same ID detected again is a new
// finding. The monotonic clock orders it exactly against the run's start. A
// regression an earlier process recorded predates every run of this one, so it
// orders none of them, whatever the clock now reads. A result for a session of
// an earlier process, such as an action outcome, is still judged against
// LastRegressionAt.
func (s *FindingsStore) liveRunOccurrenceLocked(id string) (time.Time, bool) {
startedAt, ok := s.occurrenceStarts[id]
return startedAt, ok
}
func (s *FindingsStore) updateInvestigationOutcomeLocked(f *Finding, outcome string) {
prevOutcome := f.InvestigationOutcome
f.InvestigationOutcome = outcome
// Replaying a persisted result must not invent a later recovery event.
if InvestigationOutcome(outcome) == InvestigationOutcomeFixVerified {
s.resolveVerifiedFindingLocked(f, prevOutcome)
} else if prevOutcome != outcome {
msg := "Investigation outcome updated"
if InvestigationOutcome(outcome) == InvestigationOutcomeFixFailed || InvestigationOutcome(outcome) == InvestigationOutcomeFixVerificationFailed {
msg = "Remediation failed verification"
}
s.appendLifecycleLocked(f, "investigation_outcome", msg, f.LoopState, f.LoopState, map[string]string{
"prev_outcome": prevOutcome,
"next_outcome": outcome,
})
}
s.syncLoopStateLocked(f)
}
// UpdateInvestigation updates all investigation fields on a finding
func (s *FindingsStore) UpdateInvestigation(id, sessionID, status, outcome string, lastInvestigatedAt *time.Time, attempts int) bool {
s.mu.Lock()
f, exists := s.findings[id]
if !exists {
s.mu.Unlock()
return false
}
s.updateInvestigationLocked(f, sessionID, status, outcome, lastInvestigatedAt, attempts)
s.mu.Unlock()
s.scheduleSave()
return true
}
// StartInvestigation installs a run that is starting on a finding. A run whose
// session started before the finding's current occurrence began, as
// liveRunOccurrenceLocked tells it, was started for an earlier occurrence, so
// it is not installed: it would replace the current occurrence's investigation
// state, or a newer run's. Neither is a run whose session started longer ago
// than the store keeps occurrence starts, since it could predate one already
// forgotten. runStartedAt is the start of the run's session, or zero when the
// orchestrator holds none; such a run is installed as before.
func (s *FindingsStore) StartInvestigation(id, sessionID string, runStartedAt time.Time, status, outcome string, lastInvestigatedAt *time.Time, attempts int) bool {
s.mu.Lock()
f, exists := s.findings[id]
if !exists {
s.mu.Unlock()
return false
}
occurrenceAt, ok := s.liveRunOccurrenceLocked(id)
if !runStartedAt.IsZero() && ((ok && runStartedAt.Before(occurrenceAt)) || time.Since(runStartedAt) > occurrenceStartRetention) {
s.mu.Unlock()
return false
}
s.updateInvestigationLocked(f, sessionID, status, outcome, lastInvestigatedAt, attempts)
s.mu.Unlock()
s.scheduleSave()
return true
}
// CompleteInvestigation records how a run of this process ended: its status,
// outcome, close time and attempt count. It records them only while the finding
// still names that run, so a run that ends after its finding moved on cannot
// overwrite the newer state: the run's session must be the finding's current
// one, and the run must not have started before the finding's current
// occurrence began, as liveRunOccurrenceLocked tells it. runStartedAt is the
// start of the run's session, or zero when the orchestrator holds none, which
// cannot prove ownership once this process has seen an occurrence begin. A run
// whose result does not count, but which the finding still shows as running,
// is closed now through RecoverStuckInvestigation, as the stuck-run sweep would
// later close it, rather than left running. A run that reports its planned fix
// as queued does not move the finding back from an outcome the fix's action
// has reached since: a run starts with no outcome, so one already present came
// from the action lifecycle during this run. It reports whether the run's
// result was recorded.
func (s *FindingsStore) CompleteInvestigation(id, sessionID string, runStartedAt time.Time, status, outcome string, lastInvestigatedAt *time.Time, attempts int) bool {
s.mu.Lock()
f, exists := s.findings[id]
if !exists || sessionID == "" || f.InvestigationSessionID != sessionID {
s.mu.Unlock()
return false
}
if occurrenceAt, ok := s.liveRunOccurrenceLocked(id); ok && (runStartedAt.IsZero() || runStartedAt.Before(occurrenceAt)) {
s.mu.Unlock()
closedAt := time.Now()
if lastInvestigatedAt != nil {
closedAt = *lastInvestigatedAt
}
s.RecoverStuckInvestigation(id, sessionID, runStartedAt, InvestigationStatusFailed, InvestigationOutcomeInterrupted, closedAt)
return false
}
if outcome == string(InvestigationOutcomeFixQueued) && f.InvestigationOutcome != "" {
outcome = f.InvestigationOutcome
}
s.updateInvestigationLocked(f, sessionID, status, outcome, lastInvestigatedAt, attempts)
s.mu.Unlock()
s.scheduleSave()
return true
}
func (s *FindingsStore) updateInvestigationLocked(f *Finding, sessionID, status, outcome string, lastInvestigatedAt *time.Time, attempts int) {
prevStatus := f.InvestigationStatus
prevOutcome := f.InvestigationOutcome
f.InvestigationSessionID = sessionID
f.InvestigationStatus = status
f.InvestigationOutcome = outcome
f.LastInvestigatedAt = lastInvestigatedAt
f.InvestigationAttempts = attempts
if InvestigationOutcome(outcome) == InvestigationOutcomeFixVerified {
s.resolveVerifiedFindingLocked(f, prevOutcome)
} else if prevStatus != status || prevOutcome != outcome {
s.appendLifecycleLocked(f, "investigation_updated", "Investigation state updated", f.LoopState, f.LoopState, map[string]string{
"prev_status": prevStatus,
"next_status": status,
"prev_outcome": prevOutcome,
"next_outcome": outcome,
})
}
s.syncLoopStateLocked(f)
}
// RecoverStuckInvestigation closes a run that is no longer live with the given
// status and outcome, but only while the finding still shows that run as
// running, so a result or a newer run that landed since the caller looked is
// never overwritten. runStartedAt is the start of the run's session, or zero
// when the orchestrator holds none. A run that started before the finding's
// current occurrence is only recorded as interrupted: its result is not this
// occurrence's. A run starts with no outcome, so an outcome already present
// came from the action lifecycle during this run; it is kept, so a queued fix
// still blocks another investigation. The run still closes with its own status:
// one cut short after it planned a fix failed, as it reads when it closes
// itself. The close becomes the finding's latest investigation time.
func (s *FindingsStore) RecoverStuckInvestigation(id, sessionID string, runStartedAt time.Time, status InvestigationStatus, outcome InvestigationOutcome, at time.Time) bool {
s.mu.Lock()
f, exists := s.findings[id]
if !exists || f.InvestigationStatus != string(InvestigationStatusRunning) || f.InvestigationSessionID != sessionID {
s.mu.Unlock()
return false
}
if !runStartedAt.IsZero() && !f.acceptsInvestigationResult(sessionID, runStartedAt) {
status, outcome = InvestigationStatusFailed, InvestigationOutcomeInterrupted
}
prevStatus := f.InvestigationStatus
prevOutcome := f.InvestigationOutcome
if prevOutcome == "" {
f.InvestigationOutcome = string(outcome)
if outcome == InvestigationOutcomeFixVerified {
s.resolveVerifiedFindingLocked(f, prevOutcome)
}
}
f.InvestigationStatus = string(status)
f.LastInvestigatedAt = &at
s.appendLifecycleLocked(f, "investigation_updated", "Investigation state updated", f.LoopState, f.LoopState, map[string]string{
"prev_status": prevStatus,
"next_status": string(status),
"prev_outcome": prevOutcome,
"next_outcome": f.InvestigationOutcome,
})
s.syncLoopStateLocked(f)
s.mu.Unlock()
s.scheduleSave()
return true
}
// resolveVerifiedFindingLocked projects the first verified resolution without
// moving its timestamp or appending another verification when history is replayed.
// A genuine regression clears ResolvedAt and can therefore resolve again.
func (s *FindingsStore) resolveVerifiedFindingLocked(f *Finding, previousOutcome string) {
unresolved := f.ResolvedAt == nil
if unresolved {
if f.IsActive() {
s.activeCounts[f.Severity]--
}
now := time.Now()
f.ResolvedAt = &now
}
f.AutoResolved = true
if f.ResolveReason == "" {
f.ResolveReason = "Fix verified"
}
if unresolved || InvestigationOutcome(previousOutcome) != InvestigationOutcomeFixVerified {
s.appendLifecycleLocked(f, "verification_passed", "Fix verified; finding resolved", f.LoopState, string(FindingLoopStateResolved), nil)
}
}
// Suppress marks a finding type as permanently suppressed for a resource
// Future findings with the same resource+category will be auto-dismissed
func (s *FindingsStore) Suppress(id string) bool {
s.mu.Lock()
f, exists := s.findings[id]
if !exists {
s.mu.Unlock()
return false
}
f.Suppressed = true
f.DismissedReason = "suppressed"
now := time.Now()
f.AcknowledgedAt = &now
s.appendLifecycleLocked(f, "suppressed", "Permanently suppressed by user", f.LoopState, string(FindingLoopStateSuppressed), nil)
s.syncLoopStateLocked(f)
// Create a suppression rule to block future findings with same resource+category
s.addSuppressionRuleInternal(f.ResourceID, f.ResourceName, f.Category, "Suppressed via finding", "suppress")
s.mu.Unlock()
s.scheduleSave()
return true
}
// IsSuppressed checks if findings of this type for this resource are suppressed
// Checks by resource+category only (not title, since LLM titles vary)
func (s *FindingsStore) IsSuppressed(resourceID string, category FindingCategory) bool {
s.mu.RLock()
defer s.mu.RUnlock()
return s.isSuppressedInternal(resourceID, category)
}
// isSuppressedInternal checks suppression without locking (caller must hold lock)
// IMPORTANT: Only explicit manual suppression rules block new findings.
// A dismissed finding (even with "not_an_issue") does NOT block future findings.
// This ensures that if an issue recurs, it will be detected and alerted.
// For permanent suppression, users must create an explicit suppression rule.
func (s *FindingsStore) isSuppressedInternal(resourceID string, category FindingCategory) bool {
// Only check manual suppression rules - dismissed findings should NOT block new findings
// Rationale: If frigate-storage was full and user dismissed it as "fixed", we still
// want to alert if frigate-storage fills up again in the future.
for _, rule := range s.suppressionRules {
resourceMatches := rule.ResourceID == "" || rule.ResourceID == resourceID
categoryMatches := rule.Category == "" || rule.Category == category
if resourceMatches && categoryMatches {
return true
}
}
return false
}
// Get returns a finding by ID
func (s *FindingsStore) Get(id string) *Finding {
s.mu.RLock()
defer s.mu.RUnlock()
if f, exists := s.findings[id]; exists {
// Return a copy to prevent mutations
copy := *f
return ©
}
return nil
}
// GetByResource returns all active findings for a resource
func (s *FindingsStore) GetByResource(resourceID string) []*Finding {
s.mu.RLock()
defer s.mu.RUnlock()
ids := s.byResource[resourceID]
result := make([]*Finding, 0, len(ids))
for _, id := range ids {
if f, exists := s.findings[id]; exists && f.IsActive() {
copy := *f
result = append(result, ©)
}
}
return result
}
// GetActive returns all active findings, optionally filtered by severity
func (s *FindingsStore) GetActive(minSeverity FindingSeverity) []*Finding {
s.mu.RLock()
defer s.mu.RUnlock()
severityOrder := map[FindingSeverity]int{
FindingSeverityInfo: 0,
FindingSeverityWatch: 1,
FindingSeverityWarning: 2,
FindingSeverityCritical: 3,
}
minOrder := severityOrder[minSeverity]
result := make([]*Finding, 0)
for _, f := range s.findings {
if f.IsActive() && severityOrder[f.Severity] >= minOrder {
copy := *f
result = append(result, ©)
}
}
return result
}
// GetSummary returns a summary of active findings
// Note: This calculates counts dynamically to handle time-based snooze expiration
func (s *FindingsStore) GetSummary() FindingsSummary {
s.mu.RLock()
defer s.mu.RUnlock()
summary := FindingsSummary{
Total: len(s.findings),
}
// Calculate active counts dynamically since IsActive() checks time-based snooze
for _, f := range s.findings {
if f.IsActive() {
switch f.Severity {
case FindingSeverityCritical:
summary.Critical++
case FindingSeverityWarning:
summary.Warning++
case FindingSeverityWatch:
summary.Watch++
case FindingSeverityInfo:
summary.Info++
}
}
}
return summary
}
// GetAll returns all findings including resolved ones (for history)
// Results can be filtered by time range using startTime parameter
func (s *FindingsStore) GetAll(startTime *time.Time) []*Finding {
s.mu.RLock()
defer s.mu.RUnlock()
result := make([]*Finding, 0, len(s.findings))
for _, f := range s.findings {
// If startTime specified, only include findings detected after it
if startTime != nil && f.DetectedAt.Before(*startTime) {
continue
}
copy := *f
result = append(result, ©)
}
return result
}
// ClearAll removes all findings from the store
// Returns the number of findings removed
func (s *FindingsStore) ClearAll() int {
s.mu.Lock()
count := len(s.findings)
now := time.Now()
for id := range s.findings {
s.occurrenceStarts[id] = now
}
s.findings = make(map[string]*Finding)
s.byResource = make(map[string][]string)
s.activeCounts = make(map[FindingSeverity]int)
s.mu.Unlock()
s.scheduleSave()
return count
}
// Cleanup removes old resolved findings (and trims stale dismissed history).
func (s *FindingsStore) Cleanup(maxAge time.Duration) int {
s.mu.Lock()
defer s.mu.Unlock()
now := time.Now()
cutoff := now.Add(-maxAge)
dismissedCutoff := now.Add(-30 * 24 * time.Hour)
removed := 0
for id, f := range s.findings {
shouldRemove := false
// Remove old resolved findings
if f.ResolvedAt != nil && f.ResolvedAt.Before(cutoff) {
shouldRemove = true
}
// Trim stale dismissed findings, but retain suppressed ones for memory.
if f.DismissedReason != "" && !f.Suppressed && f.LastSeenAt.Before(dismissedCutoff) {
shouldRemove = true
}
if shouldRemove {
delete(s.findings, id)
s.occurrenceStarts[id] = now
// Clean up resource index
ids := s.byResource[f.ResourceID]
for i, fid := range ids {
if fid == id {
s.byResource[f.ResourceID] = append(ids[:i], ids[i+1:]...)
break
}
}
removed++
}
}
for id, startedAt := range s.occurrenceStarts {
if startedAt.Before(now.Add(-occurrenceStartRetention)) {
delete(s.occurrenceStarts, id)
}
}
return removed
}
// GetDismissedForContext returns findings that the user has dismissed/acknowledged,
// formatted for injection into LLM prompts. This is the core of the "memory" system -
// it tells the LLM what not to re-raise.
func (s *FindingsStore) GetDismissedForContext() string {
s.mu.RLock()
defer s.mu.RUnlock()
return s.dismissedContextLocked(nil)
}
// GetDismissedForContextForResources returns dismissed/suppressed/snoozed findings
// limited to the provided resource IDs/names. An empty filter behaves like the
// unscoped context and returns all eligible findings.
func (s *FindingsStore) GetDismissedForContextForResources(resourceFilter map[string]bool) string {
s.mu.RLock()
defer s.mu.RUnlock()
return s.dismissedContextLocked(resourceFilter)
}
func (s *FindingsStore) dismissedContextLocked(resourceFilter map[string]bool) string {
var suppressed, dismissed, snoozed []string
includeFinding := func(f *Finding) bool {
if len(resourceFilter) == 0 {
return true
}
return resourceFilter[f.ResourceID] || resourceFilter[f.ResourceName]
}
for _, f := range s.findings {
if !includeFinding(f) {
continue
}
// Collect suppressed findings
if f.Suppressed {
note := ""
if f.UserNote != "" {
note = " - User note: " + f.UserNote
}
suppressed = append(suppressed,
fmt.Sprintf("- %s on %s: %s%s", f.Title, f.ResourceName, f.DismissedReason, note))
continue
}
// Skip very old findings (more than 30 days)
if time.Since(f.LastSeenAt) > 30*24*time.Hour {
continue
}
// Collect dismissed/acknowledged findings
if f.DismissedReason != "" {
note := ""
if f.UserNote != "" {
note = " - User note: " + f.UserNote
}
dismissed = append(dismissed,
fmt.Sprintf("- %s on %s (%s)%s", f.Title, f.ResourceName, f.DismissedReason, note))
continue
}
// Collect snoozed findings
if f.IsSnoozed() {
snoozed = append(snoozed,
fmt.Sprintf("- %s on %s (snoozed until %s)",
f.Title, f.ResourceName, f.SnoozedUntil.Format("Jan 2")))
}
}
if len(suppressed) == 0 && len(dismissed) == 0 && len(snoozed) == 0 {
return ""
}
var result strings.Builder
result.WriteString("\n## Previous Findings - User Feedback\n")
result.WriteString("The following findings have been addressed by the user. Do NOT re-raise these unless the situation has significantly worsened:\n\n")
if len(suppressed) > 0 {
result.WriteString("### Permanently Suppressed (never re-raise):\n")
for _, s := range suppressed {
result.WriteString(s + "\n")
}
result.WriteString("\n")
}
if len(dismissed) > 0 {
result.WriteString("### Dismissed by User:\n")
for _, d := range dismissed {
result.WriteString(d + "\n")
}
result.WriteString("\n")
}
if len(snoozed) > 0 {
result.WriteString("### Temporarily Snoozed:\n")
for _, s := range snoozed {
result.WriteString(s + "\n")
}
}
return result.String()
}
// FindingsSummary provides a quick count of findings by severity
type FindingsSummary struct {
Critical int `json:"critical"`
Warning int `json:"warning"`
Watch int `json:"watch"`
Info int `json:"info"`
Total int `json:"total"`
}
// FindingsTrustSummary is a snapshot of how the currently-tracked findings
// have resolved over time. It answers the user-facing "do I trust Patrol?"
// question with concrete counts: how many were auto-resolved, how many were
// fix-verified, how many were dismissed as noise versus expected behavior.
//
// This is a snapshot of the in-memory store; it does not include historical
// findings that have been cleaned up. Treat the counts as "current state
// distribution of tracked findings," not lifetime totals.
type FindingsTrustSummary struct {
Tracked int `json:"tracked"`
CurrentlyActive int `json:"currently_active"`
Resolved int `json:"resolved"`
AutoResolved int `json:"auto_resolved"`
FixVerified int `json:"fix_verified"`
FixFailed int `json:"fix_failed"`
DismissedAsNoise int `json:"dismissed_as_noise"` // dismissed_reason=not_an_issue
DismissedAsExpected int `json:"dismissed_as_expected"` // dismissed_reason=expected_behavior
DismissedAsLater int `json:"dismissed_as_later"` // dismissed_reason=will_fix_later
Suppressed int `json:"suppressed"`
RegressedAtLeastOnce int `json:"regressed_at_least_once"`
}
// HasIssues returns true if there are any warning or critical findings
func (s FindingsSummary) HasIssues() bool {
return s.Critical > 0 || s.Warning > 0
}
// GetTrustSummary returns a snapshot of how currently-tracked findings have
// resolved. See FindingsTrustSummary for caveats on snapshot vs lifetime
// totals.
func (s *FindingsStore) GetTrustSummary() FindingsTrustSummary {
s.mu.RLock()
defer s.mu.RUnlock()
var summary FindingsTrustSummary
for _, f := range s.findings {
summary.Tracked++
if f.IsActive() {
summary.CurrentlyActive++
}
if f.ResolvedAt != nil {
summary.Resolved++
if f.AutoResolved {
summary.AutoResolved++
}
}
if f.Suppressed {
summary.Suppressed++
}
if f.RegressionCount > 0 {
summary.RegressedAtLeastOnce++
}
switch f.InvestigationOutcome {
case string(aicontracts.OutcomeFixVerified):
summary.FixVerified++
case string(aicontracts.OutcomeFixFailed),
string(aicontracts.OutcomeFixVerificationFailed):
summary.FixFailed++
}
switch f.DismissedReason {
case "not_an_issue":
summary.DismissedAsNoise++
case "expected_behavior":
summary.DismissedAsExpected++
case "will_fix_later":
summary.DismissedAsLater++
}
}
return summary
}
// IsHealthy returns true if there are no watch, warning, or critical findings
func (s FindingsSummary) IsHealthy() bool {
return s.Critical == 0 && s.Warning == 0 && s.Watch == 0
}
// --- Suppression Rule Management ---
// AddSuppressionRule creates a new user-defined suppression rule
func (s *FindingsStore) AddSuppressionRule(resourceID, resourceName string, category FindingCategory, description string) *SuppressionRule {
s.mu.Lock()
rule := s.addSuppressionRuleInternal(resourceID, resourceName, category, description, "manual")
s.mu.Unlock()
s.scheduleSave()
return rule
}
// addSuppressionRuleInternal creates a suppression rule without locking (caller must hold lock)
func (s *FindingsStore) addSuppressionRuleInternal(resourceID, resourceName string, category FindingCategory, description, createdFrom string) *SuppressionRule {
// Generate ID based on resource+category
ruleID := fmt.Sprintf("rule_%s_%s_%d", resourceID, category, time.Now().UnixNano())
if resourceID == "" {
ruleID = fmt.Sprintf("rule_any_%s_%d", category, time.Now().UnixNano())
}
if category == "" {
ruleID = fmt.Sprintf("rule_%s_any_%d", resourceID, time.Now().UnixNano())
}
rule := &SuppressionRule{
ID: ruleID,
ResourceID: resourceID,
ResourceName: resourceName,
Category: category,
Description: description,
CreatedAt: time.Now(),
CreatedFrom: createdFrom,
}
s.suppressionRules[ruleID] = rule
return rule
}
// GetSuppressionRules returns all suppression rules (both manual and from dismissed findings)
func (s *FindingsStore) GetSuppressionRules() []*SuppressionRule {
s.mu.RLock()
defer s.mu.RUnlock()
var rules []*SuppressionRule
// Add explicit suppression rules
for _, rule := range s.suppressionRules {
rules = append(rules, rule)
}
// Include all dismissed findings as rules (for visibility and to allow reverting)
// This covers both suppressed ("not_an_issue") and acknowledged ("expected_behavior", "will_fix_later")
for _, f := range s.findings {
if f.Suppressed || f.DismissedReason != "" {
createdFrom := "finding"
if !f.Suppressed && f.DismissedReason != "" {
// Mark as "dismissed" type to distinguish from permanent suppression
createdFrom = "dismissed"
}
// Handle nil AcknowledgedAt (shouldn't happen but be safe)
createdAt := f.LastSeenAt
if f.AcknowledgedAt != nil {
createdAt = *f.AcknowledgedAt
}
rules = append(rules, &SuppressionRule{
ID: "finding_" + f.ID,
ResourceID: f.ResourceID,
ResourceName: f.ResourceName,
Category: f.Category,
Description: f.UserNote,
DismissedReason: f.DismissedReason,
CreatedAt: createdAt,
CreatedFrom: createdFrom,
FindingID: f.ID,
})
}
}
return rules
}
// DeleteSuppressionRule removes a suppression rule
func (s *FindingsStore) DeleteSuppressionRule(ruleID string) bool {
s.mu.Lock()
// Check if it's an explicit rule
if _, exists := s.suppressionRules[ruleID]; exists {
delete(s.suppressionRules, ruleID)
s.mu.Unlock()
s.scheduleSave()
return true
}
// Check if it's a finding-based rule (e.g., "finding_abc123")
if strings.HasPrefix(ruleID, "finding_") {
findingID := strings.TrimPrefix(ruleID, "finding_")
if f, exists := s.findings[findingID]; exists && (f.Suppressed || f.DismissedReason != "") {
// Finding-backed suppression rows are the management surface for
// dismissed findings. Use the canonical transition so reopening
// also clears RemindAt and records an undismissed lifecycle event.
s.undismissLocked(f)
s.mu.Unlock()
s.scheduleSave()
return true
}
}
s.mu.Unlock()
return false
}
// GetDismissedFindings returns all findings that have been dismissed or suppressed
func (s *FindingsStore) GetDismissedFindings() []*Finding {
s.mu.RLock()
defer s.mu.RUnlock()
var dismissed []*Finding
for _, f := range s.findings {
if f.DismissedReason != "" || f.Suppressed {
copy := *f
dismissed = append(dismissed, ©)
}
}
return dismissed
}
// MatchesSuppressionRule checks if a finding matches any suppression rule
func (s *FindingsStore) MatchesSuppressionRule(resourceID string, category FindingCategory) bool {
s.mu.RLock()
defer s.mu.RUnlock()
for _, rule := range s.suppressionRules {
// Check if resource matches (or rule is for "any resource")
resourceMatches := rule.ResourceID == "" || rule.ResourceID == resourceID
// Check if category matches (or rule is for "any category")
categoryMatches := rule.Category == "" || rule.Category == category
if resourceMatches && categoryMatches {
return true
}
}
return false
}
// --- Semantic Deduplication ---
// FindingCluster represents a group of semantically related findings
type FindingCluster struct {
ID string `json:"id"`
PrimaryFindingID string `json:"primary_finding_id"` // The "main" finding in the cluster
RelatedIDs []string `json:"related_ids"` // Other findings in the cluster
CommonCategory FindingCategory `json:"common_category"`
CommonResource string `json:"common_resource,omitempty"` // If all findings are for same resource
Summary string `json:"summary"` // Aggregated summary
HighestSeverity FindingSeverity `json:"highest_severity"`
TotalCount int `json:"total_count"`
}
// SemanticSimilarity calculates similarity between two findings (0-1)
func SemanticSimilarity(f1, f2 *Finding) float64 {
if f1 == nil || f2 == nil {
return 0
}
var similarity float64
// Same resource is a strong signal
if f1.ResourceID == f2.ResourceID {
similarity += 0.3
}
// Same category
if f1.Category == f2.Category {
similarity += 0.2
}
// Similar key (if set)
if f1.Key != "" && f1.Key == f2.Key {
similarity += 0.4
}
// Title keyword overlap
titleSim := keywordOverlap(f1.Title, f2.Title)
similarity += titleSim * 0.2
// Description keyword overlap
descSim := keywordOverlap(f1.Description, f2.Description)
similarity += descSim * 0.1
return minFloat(similarity, 1.0)
}
// keywordOverlap calculates the Jaccard similarity of keywords between two strings
func keywordOverlap(s1, s2 string) float64 {
words1 := extractKeywords(s1)
words2 := extractKeywords(s2)
if len(words1) == 0 || len(words2) == 0 {
return 0
}
// Calculate intersection
intersection := 0
for word := range words1 {
if words2[word] {
intersection++
}
}
// Calculate union
union := len(words1)
for word := range words2 {
if !words1[word] {
union++
}
}
if union == 0 {
return 0
}
return float64(intersection) / float64(union)
}
// extractKeywords extracts significant keywords from a string
func extractKeywords(s string) map[string]bool {
keywords := make(map[string]bool)
// Common stop words to ignore
stopWords := map[string]bool{
"the": true, "a": true, "an": true, "is": true, "are": true,
"was": true, "were": true, "be": true, "been": true, "being": true,
"have": true, "has": true, "had": true, "do": true, "does": true,
"did": true, "will": true, "would": true, "could": true, "should": true,
"may": true, "might": true, "must": true, "shall": true, "can": true,
"to": true, "of": true, "in": true, "for": true, "on": true, "with": true,
"at": true, "by": true, "from": true, "as": true, "into": true, "through": true,
"and": true, "or": true, "but": true, "if": true, "then": true, "than": true,
"this": true, "that": true, "these": true, "those": true, "it": true,
}
// Split into words and normalize
word := ""
for _, c := range s {
if (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') {
if c >= 'A' && c <= 'Z' {
c = c + 32 // lowercase
}
word += string(c)
} else {
if len(word) > 2 && !stopWords[word] {
keywords[word] = true
}
word = ""
}
}
if len(word) > 2 && !stopWords[word] {
keywords[word] = true
}
return keywords
}
// FindSimilarFindings finds findings similar to the given one
func (s *FindingsStore) FindSimilarFindings(f *Finding, minSimilarity float64) []*Finding {
s.mu.RLock()
defer s.mu.RUnlock()
var similar []*Finding
for _, existing := range s.findings {
if existing.ID == f.ID {
continue
}
if !existing.IsActive() {
continue
}
similarity := SemanticSimilarity(f, existing)
if similarity >= minSimilarity {
copy := *existing
similar = append(similar, ©)
}
}
return similar
}
// AddWithDeduplication adds a finding, merging with similar existing findings if found
// Returns the finding ID (may be existing finding if merged) and whether it was new
func (s *FindingsStore) AddWithDeduplication(f *Finding, minSimilarity float64) (string, bool) {
// First check for similar findings
similar := s.FindSimilarFindings(f, minSimilarity)
if len(similar) > 0 {
// Find the most similar one
var bestMatch *Finding
var bestSimilarity float64
for _, existing := range similar {
sim := SemanticSimilarity(f, existing)
if sim > bestSimilarity {
bestMatch = existing
bestSimilarity = sim
}
}
if bestMatch != nil {
// Merge with existing finding
s.mu.Lock()
if existing, ok := s.findings[bestMatch.ID]; ok {
existing.LastSeenAt = time.Now()
existing.TimesRaised++
// Update to higher severity if new finding has higher severity
severityOrder := map[FindingSeverity]int{
FindingSeverityInfo: 0,
FindingSeverityWatch: 1,
FindingSeverityWarning: 2,
FindingSeverityCritical: 3,
}
if severityOrder[f.Severity] > severityOrder[existing.Severity] {
existing.Severity = f.Severity
}
// Append evidence if different
if f.Evidence != "" && f.Evidence != existing.Evidence {
existing.Evidence = existing.Evidence + "\n---\n" + f.Evidence
// Truncate if too long
if len(existing.Evidence) > 5000 {
existing.Evidence = existing.Evidence[:5000] + "..."
}
}
}
s.mu.Unlock()
s.scheduleSave()
return bestMatch.ID, false
}
}
// No similar finding found, add as new
isNew := s.Add(f)
return f.ID, isNew
}
// GetFindingClusters groups active findings into clusters based on semantic similarity
func (s *FindingsStore) GetFindingClusters(minSimilarity float64) []*FindingCluster {
s.mu.RLock()
defer s.mu.RUnlock()
// Get all active findings
var activeFindings []*Finding
for _, f := range s.findings {
if f.IsActive() {
copy := *f
activeFindings = append(activeFindings, ©)
}
}
if len(activeFindings) == 0 {
return nil
}
// Track which findings have been clustered
clustered := make(map[string]bool)
var clusters []*FindingCluster
// For each finding, find similar ones and create a cluster
for _, f := range activeFindings {
if clustered[f.ID] {
continue
}
cluster := &FindingCluster{
ID: fmt.Sprintf("cluster-%s", f.ID[:8]),
PrimaryFindingID: f.ID,
RelatedIDs: make([]string, 0),
CommonCategory: f.Category,
CommonResource: f.ResourceID,
HighestSeverity: f.Severity,
TotalCount: 1,
}
clustered[f.ID] = true
// Find all similar findings
for _, other := range activeFindings {
if other.ID == f.ID || clustered[other.ID] {
continue
}
similarity := SemanticSimilarity(f, other)
if similarity >= minSimilarity {
cluster.RelatedIDs = append(cluster.RelatedIDs, other.ID)
cluster.TotalCount++
clustered[other.ID] = true
// Track if all are same resource
if other.ResourceID != f.ResourceID {
cluster.CommonResource = ""
}
// Track highest severity
severityOrder := map[FindingSeverity]int{
FindingSeverityInfo: 0,
FindingSeverityWatch: 1,
FindingSeverityWarning: 2,
FindingSeverityCritical: 3,
}
if severityOrder[other.Severity] > severityOrder[cluster.HighestSeverity] {
cluster.HighestSeverity = other.Severity
}
}
}
// Generate cluster summary
if cluster.TotalCount > 1 {
cluster.Summary = fmt.Sprintf("%d related %s findings", cluster.TotalCount, cluster.CommonCategory)
if cluster.CommonResource != "" {
cluster.Summary += " for " + cluster.CommonResource
}
} else {
cluster.Summary = f.Title
}
clusters = append(clusters, cluster)
}
return clusters
}
func minFloat(a, b float64) float64 {
if a < b {
return a
}
return b
}