Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Keep stalled operator lookups out of Patrol shutdown
rcourtman47c3ddbd8a3d
diff --git a/docs/release-control/v6/internal/subsystems/ai-runtime.md b/docs/release-control/v6/internal/subsystems/ai-runtime.mdindex 73bcfa0661..4b47868c69 100644--- a/docs/release-control/v6/internal/subsystems/ai-runtime.md+++ b/docs/release-control/v6/internal/subsystems/ai-runtime.md@@ -4281,6 +4281,20 @@ findings. The field persists as `resource_criticality` and may affectsame-severity Patrol attention ordering only; it must not mutate`Finding.Severity` or bypass severity escalation/resolution rules.+The operator-state projection runs outside the findings mutex. An external+adapter waiting on storage or reading findings must not prevent Patrol shutdown+from saving findings already accepted by the store. Before applying the result,+`FindingsStore.Add` rechecks the provider revision and the retained resource+identity used by identity-free updates; replacement or rebinding requires a new+projection. A pending lookup is not an accepted finding. This does not turn a+blocked external call into a completed investigation or claim its result was+flushed. `TestPatrolService_StopWhileOperatorStateLookupBlocked` exercises the+shutdown save against real config persistence; the provider replacement and+removal cases in `TestFindingsStore_AddUsesCurrentOperatorStateProvider` reject+stale suppression and allow the current provider to read the store.+`TestFindingsStore_AddRefreshesReboundOperatorResource` covers a retained+identity changing while an identity-free update is waiting on its projection.+The approval store exposes `SetOnApprovalCreated(cb)` so the APIlayer can install a fire-and-forget callback that runs after everysuccessful `CreateApproval` (the approval is already persisted atdiff --git a/internal/ai/findings.go b/internal/ai/findings.goindex 8067da065d..50ec2a861e 100644--- a/internal/ai/findings.go+++ b/internal/ai/findings.go@@ -863,7 +863,8 @@ type FindingsStore struct {// other operator-set intent, auto-acknowledging new findings whose// resource is in a maintenance window. Nil-safe; an unset provider// disables operator-state-driven suppression.- resourceOperatorStateProvider ResourceOperatorStateProvider+ resourceOperatorStateProvider ResourceOperatorStateProvider+ resourceOperatorStateProviderRevision uint64// Optional finding-storm throttler — when set, the store invokes it// from Add's new-finding branch so a single noisy resource's burst// of multi-symptom findings is summarised by one storm finding.@@ -986,6 +987,7 @@ func (s *FindingsStore) SetResourceOperatorStateProvider(p ResourceOperatorStates.mu.Lock()defer s.mu.Unlock()s.resourceOperatorStateProvider = p+ s.resourceOperatorStateProviderRevision++}// SetStormThrottler installs the finding-storm throttler. The store@@ -1906,6 +1908,18 @@ func (s *FindingsStore) syncLoopStateLocked(f *Finding) {// it's the only signal that a transition was rejected, not a duplicate.}+// operatorResourceIDLocked resolves Add's operator-state lookup identity.+// Caller holds s.mu; an update without an identity uses the retained finding.+func (s *FindingsStore) operatorResourceIDLocked(f *Finding) string {+ if id := strings.TrimSpace(f.ResourceID); id != "" {+ return id+ }+ if existing := s.findings[f.ID]; existing != nil {+ return strings.TrimSpace(existing.ResourceID)+ }+ return ""+}+// Add adds or updates a finding// If a finding with the same ID exists, it updates LastSeenAt and increments TimesRaised// If the finding is suppressed or dismissed, it may be skipped@@ -1913,6 +1927,32 @@ func (s *FindingsStore) syncLoopStateLocked(f *Finding) {func (s *FindingsStore) Add(f *Finding) bool {s.mu.Lock()+ // Operator-state adapters can wait on storage or read this store. Do not+ // retain the findings lock across that external call: shutdown needs it to+ // persist findings already accepted, even if a new lookup is stalled.+ var operatorProjection ResourceOperatorStateProjection+ operatorProjectionOK := false+ operatorStateProviderPresent := false+ for {+ provider := s.resourceOperatorStateProvider+ resourceID := s.operatorResourceIDLocked(f)+ if provider == nil || resourceID == "" {+ break+ }+ revision := s.resourceOperatorStateProviderRevision+ s.mu.Unlock()+ projection, ok := provider.OperatorStateProjection(resourceID, time.Now())+ s.mu.Lock()+ // A replaced provider or rebound fallback identity invalidates the+ // result. Re-read before committing suppression or criticality.+ if revision != s.resourceOperatorStateProviderRevision || resourceID != s.operatorResourceIDLocked(f) {+ continue+ }+ operatorProjection, operatorProjectionOK = projection, ok+ operatorStateProviderPresent = true+ break+ }+if f.ResourceType == "" {f.ResourceType = inferFindingResourceType(f.ResourceID, f.ResourceName)}@@ -1920,24 +1960,13 @@ func (s *FindingsStore) Add(f *Finding) bool {f.syncLoopState()existing, exists := s.findings[f.ID]- operatorStateProviderPresent := false- operatorProjectionOK := false- var operatorProjection ResourceOperatorStateProjection- if provider := s.resourceOperatorStateProvider; provider != nil {- operatorResourceID := strings.TrimSpace(f.ResourceID)- if operatorResourceID == "" && existing != nil {- operatorResourceID = strings.TrimSpace(existing.ResourceID)- }- if operatorResourceID != "" {- operatorStateProviderPresent = true- operatorProjection, operatorProjectionOK = provider.OperatorStateProjection(operatorResourceID, time.Now())- if operatorProjectionOK {- f.ResourceCriticality = normalizeResourceCriticality(operatorProjection.Criticality)- } else {- f.ResourceCriticality = ""- }+ if operatorStateProviderPresent {+ if operatorProjectionOK {+ f.ResourceCriticality = normalizeResourceCriticality(operatorProjection.Criticality)+ } else {+ f.ResourceCriticality = ""}- } else {+ } else if s.resourceOperatorStateProvider == nil {f.ResourceCriticality = normalizeResourceCriticality(f.ResourceCriticality)}if exists {diff --git a/internal/ai/findings_test.go b/internal/ai/findings_test.goindex cb2f2c6a6d..ea1606f8d8 100644--- a/internal/ai/findings_test.go+++ b/internal/ai/findings_test.go@@ -1711,6 +1711,145 @@ func TestFinding_RemindAt_RoundTripsThroughJSON(t *testing.T) {}}+func TestFindingsStore_AddUsesCurrentOperatorStateProvider(t *testing.T) {+ for _, replace := range []bool{false, true} {+ name := "remove"+ if replace {+ name = "replace"+ }+ t.Run(name, func(t *testing.T) {+ store := NewFindingsStore()+ entered, release := make(chan struct{}), make(chan struct{})+ store.SetResourceOperatorStateProvider(ResourceOperatorStateProviderFunc(func(string, time.Time) (ResourceOperatorStateProjection, bool) {+ close(entered)+ <-release+ return ResourceOperatorStateProjection{IntentionallyOffline: true, Criticality: "high"}, true+ }))+ addDone, changed := make(chan struct{}), make(chan struct{})+ go func() {+ store.Add(&Finding{ID: "f1", ResourceID: "vm:101", Title: "Unavailable", Severity: FindingSeverityInfo})+ close(addDone)+ }()+ defer func() {+ close(release)+ select {+ case <-changed:+ case <-time.After(3 * time.Second):+ t.Error("provider change did not finish")+ }+ select {+ case <-addDone:+ case <-time.After(3 * time.Second):+ t.Error("finding add did not finish")+ }+ }()+ select {+ case <-entered:+ case <-time.After(3 * time.Second):+ t.Fatal("operator-state lookup did not start")+ }+ go func() {+ var next ResourceOperatorStateProvider+ if replace {+ next = ResourceOperatorStateProviderFunc(func(string, time.Time) (ResourceOperatorStateProjection, bool) {+ // A provider may read the store while projecting state.+ store.GetSummary()+ return ResourceOperatorStateProjection{Criticality: "low"}, true+ })+ }+ store.SetResourceOperatorStateProvider(next)+ close(changed)+ }()+ select {+ case <-changed:+ case <-time.After(3 * time.Second):+ t.Error("provider cannot be replaced while its previous lookup is blocked")+ return+ }+ // Release the old result, then wait for the new projection to commit.+ release <- struct{}{}+ select {+ case <-addDone:+ case <-time.After(3 * time.Second):+ t.Error("provider re-entry blocked finding add")+ return+ }+ got := store.Get("f1")+ if got == nil || got.DismissedReason != "" {+ t.Fatalf("stale provider suppressed the finding: %+v", got)+ }+ want := ""+ if replace {+ want = "low"+ }+ if got.ResourceCriticality != want {+ t.Errorf("criticality = %q, want %q from current provider", got.ResourceCriticality, want)+ }+ })+ }+}++func TestFindingsStore_AddRefreshesReboundOperatorResource(t *testing.T) {+ store := NewFindingsStore()+ store.Add(&Finding{ID: "f1", ResourceID: "vm:100", Title: "Unavailable", Severity: FindingSeverityInfo})+ entered, release := make(chan struct{}), make(chan struct{})+ store.SetResourceOperatorStateProvider(ResourceOperatorStateProviderFunc(func(id string, _ time.Time) (ResourceOperatorStateProjection, bool) {+ if id == "vm:100" {+ close(entered)+ <-release+ return ResourceOperatorStateProjection{IntentionallyOffline: true, Criticality: "high"}, true+ }+ return ResourceOperatorStateProjection{Criticality: "low"}, id == "vm:101"+ }))+ addDone, rebound := make(chan struct{}), make(chan error, 1)+ go func() {+ store.Add(&Finding{ID: "f1", Title: "Unavailable", Severity: FindingSeverityInfo})+ close(addDone)+ }()+ defer func() {+ close(release)+ select {+ case <-addDone:+ case <-time.After(3 * time.Second):+ t.Error("finding add did not finish")+ return+ }+ if err := store.ForceSave(); err != nil {+ t.Error(err)+ }+ }()+ select {+ case <-entered:+ case <-time.After(3 * time.Second):+ t.Fatal("operator-state lookup did not start")+ }+ go func() {+ rebound <- store.SetPersistence(&recordingPersistence{findings: map[string]*Finding{+ "f1": {ID: "f1", ResourceID: "vm:101", Title: "Unavailable", Severity: FindingSeverityInfo},+ }})+ }()+ select {+ case err := <-rebound:+ if err != nil {+ t.Fatal(err)+ }+ case <-time.After(3 * time.Second):+ t.Error("retained identity cannot change during an external lookup")+ return+ }+ release <- struct{}{}+ select {+ case <-addDone:+ case <-time.After(3 * time.Second):+ t.Error("finding add did not finish after identity changed")+ return+ }+ got := store.Get("f1")+ if got == nil || got.ResourceID != "vm:101" || got.ResourceCriticality != "low" || got.DismissedReason != "" {+ t.Fatalf("finding committed a stale resource projection: %+v", got)+ }+}+func TestFindingsStore_Add_AutoAcknowledgesDuringMaintenanceWindow(t *testing.T) {// When the operator has set a maintenance window covering the// finding's resource, the new-finding path must auto-dismiss thediff --git a/internal/ai/patrol_test.go b/internal/ai/patrol_test.goindex a84569227e..f8944eec88 100644--- a/internal/ai/patrol_test.go+++ b/internal/ai/patrol_test.go@@ -7,6 +7,7 @@ import ("time""github.com/rcourtman/pulse-go-rewrite/internal/ai/memory"+ "github.com/rcourtman/pulse-go-rewrite/internal/config""github.com/rcourtman/pulse-go-rewrite/internal/models"ur "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources")@@ -1089,6 +1090,68 @@ func TestPatrolService_GetIntelligence(t *testing.T) {}}+func TestPatrolService_StopWhileOperatorStateLookupBlocked(t *testing.T) {+ ps := NewPatrolService(nil, nil)+ persistence := config.NewConfigPersistence(t.TempDir())+ if err := ps.SetFindingsPersistence(NewFindingsPersistenceAdapter(persistence)); err != nil {+ t.Fatal(err)+ }+ ps.findings.Add(&Finding{ID: "retained", ResourceID: "vm:100", Title: "Retained finding", Severity: FindingSeverityWarning})+ entered, release := make(chan struct{}), make(chan struct{})+ ps.findings.SetResourceOperatorStateProvider(ResourceOperatorStateProviderFunc(func(string, time.Time) (ResourceOperatorStateProjection, bool) {+ close(entered)+ <-release+ return ResourceOperatorStateProjection{}, false+ }))+ addDone, stopDone := make(chan struct{}), make(chan struct{})+ go func() {+ ps.findings.Add(&Finding{ID: "pending", ResourceID: "vm:101", Title: "Pending lookup", Severity: FindingSeverityInfo})+ close(addDone)+ }()+ defer func() {+ close(release)+ select {+ case <-addDone:+ case <-time.After(3 * time.Second):+ t.Error("finding add did not finish after its lookup was released")+ return+ }+ select {+ case <-stopDone:+ case <-time.After(3 * time.Second):+ t.Error("Patrol stop did not finish after its lookup was released")+ return+ }+ if err := ps.findings.ForceSave(); err != nil {+ t.Error(err)+ }+ }()+ select {+ case <-entered:+ case <-time.After(3 * time.Second):+ t.Fatal("operator-state lookup did not start")+ }+ ps.running = true+ ps.stopCh = make(chan struct{})+ go func() { ps.Stop(); close(stopDone) }()+ select {+ case <-stopDone:+ case <-time.After(3 * time.Second):+ t.Error("Patrol shutdown cannot save retained findings while an external operator-state lookup is blocked")+ return+ }+ saved, err := persistence.LoadAIFindings()+ if err != nil {+ t.Fatal(err)+ }+ if saved.Findings["retained"] == nil {+ t.Error("shutdown did not preserve the retained finding")+ }+ if saved.Findings["pending"] != nil {+ t.Error("shutdown saved a finding whose operator-state lookup had not completed")+ }+}+func TestPatrolService_Stop(t *testing.T) {ps := NewPatrolService(nil, nil)