Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Keep stalled operator lookups out of Patrol shutdown

rcourtman47c3ddbd8a3d
diff --git a/docs/release-control/v6/internal/subsystems/ai-runtime.md b/docs/release-control/v6/internal/subsystems/ai-runtime.md
index 73bcfa0661..4b47868c69 100644
--- a/docs/release-control/v6/internal/subsystems/ai-runtime.md
+++ b/docs/release-control/v6/internal/subsystems/ai-runtime.md
@@ -4281,6 +4281,20 @@ findings. The field persists as `resource_criticality` and may affect
same-severity Patrol attention ordering only; it must not mutate
`Finding.Severity` or bypass severity escalation/resolution rules.
+The operator-state projection runs outside the findings mutex. An external
+adapter waiting on storage or reading findings must not prevent Patrol shutdown
+from saving findings already accepted by the store. Before applying the result,
+`FindingsStore.Add` rechecks the provider revision and the retained resource
+identity used by identity-free updates; replacement or rebinding requires a new
+projection. A pending lookup is not an accepted finding. This does not turn a
+blocked external call into a completed investigation or claim its result was
+flushed. `TestPatrolService_StopWhileOperatorStateLookupBlocked` exercises the
+shutdown save against real config persistence; the provider replacement and
+removal cases in `TestFindingsStore_AddUsesCurrentOperatorStateProvider` reject
+stale suppression and allow the current provider to read the store.
+`TestFindingsStore_AddRefreshesReboundOperatorResource` covers a retained
+identity changing while an identity-free update is waiting on its projection.
+
The approval store exposes `SetOnApprovalCreated(cb)` so the API
layer can install a fire-and-forget callback that runs after every
successful `CreateApproval` (the approval is already persisted at
diff --git a/internal/ai/findings.go b/internal/ai/findings.go
index 8067da065d..50ec2a861e 100644
--- a/internal/ai/findings.go
+++ b/internal/ai/findings.go
@@ -863,7 +863,8 @@ type FindingsStore struct {
// other operator-set intent, auto-acknowledging new findings whose
// resource is in a maintenance window. Nil-safe; an unset provider
// disables operator-state-driven suppression.
- resourceOperatorStateProvider ResourceOperatorStateProvider
+ resourceOperatorStateProvider ResourceOperatorStateProvider
+ resourceOperatorStateProviderRevision uint64
// Optional finding-storm throttler — when set, the store invokes it
// from Add's new-finding branch so a single noisy resource's burst
// of multi-symptom findings is summarised by one storm finding.
@@ -986,6 +987,7 @@ func (s *FindingsStore) SetResourceOperatorStateProvider(p ResourceOperatorState
s.mu.Lock()
defer s.mu.Unlock()
s.resourceOperatorStateProvider = p
+ s.resourceOperatorStateProviderRevision++
}
// SetStormThrottler installs the finding-storm throttler. The store
@@ -1906,6 +1908,18 @@ func (s *FindingsStore) syncLoopStateLocked(f *Finding) {
// it's the only signal that a transition was rejected, not a duplicate.
}
+// operatorResourceIDLocked resolves Add's operator-state lookup identity.
+// Caller holds s.mu; an update without an identity uses the retained finding.
+func (s *FindingsStore) operatorResourceIDLocked(f *Finding) string {
+ if id := strings.TrimSpace(f.ResourceID); id != "" {
+ return id
+ }
+ if existing := s.findings[f.ID]; existing != nil {
+ return strings.TrimSpace(existing.ResourceID)
+ }
+ return ""
+}
+
// Add adds or updates a finding
// If a finding with the same ID exists, it updates LastSeenAt and increments TimesRaised
// If the finding is suppressed or dismissed, it may be skipped
@@ -1913,6 +1927,32 @@ func (s *FindingsStore) syncLoopStateLocked(f *Finding) {
func (s *FindingsStore) Add(f *Finding) bool {
s.mu.Lock()
+ // Operator-state adapters can wait on storage or read this store. Do not
+ // retain the findings lock across that external call: shutdown needs it to
+ // persist findings already accepted, even if a new lookup is stalled.
+ var operatorProjection ResourceOperatorStateProjection
+ operatorProjectionOK := false
+ operatorStateProviderPresent := false
+ for {
+ provider := s.resourceOperatorStateProvider
+ resourceID := s.operatorResourceIDLocked(f)
+ if provider == nil || resourceID == "" {
+ break
+ }
+ revision := s.resourceOperatorStateProviderRevision
+ s.mu.Unlock()
+ projection, ok := provider.OperatorStateProjection(resourceID, time.Now())
+ s.mu.Lock()
+ // A replaced provider or rebound fallback identity invalidates the
+ // result. Re-read before committing suppression or criticality.
+ if revision != s.resourceOperatorStateProviderRevision || resourceID != s.operatorResourceIDLocked(f) {
+ continue
+ }
+ operatorProjection, operatorProjectionOK = projection, ok
+ operatorStateProviderPresent = true
+ break
+ }
+
if f.ResourceType == "" {
f.ResourceType = inferFindingResourceType(f.ResourceID, f.ResourceName)
}
@@ -1920,24 +1960,13 @@ func (s *FindingsStore) Add(f *Finding) bool {
f.syncLoopState()
existing, exists := s.findings[f.ID]
- operatorStateProviderPresent := false
- operatorProjectionOK := false
- var operatorProjection ResourceOperatorStateProjection
- if provider := s.resourceOperatorStateProvider; provider != nil {
- operatorResourceID := strings.TrimSpace(f.ResourceID)
- if operatorResourceID == "" && existing != nil {
- operatorResourceID = strings.TrimSpace(existing.ResourceID)
- }
- if operatorResourceID != "" {
- operatorStateProviderPresent = true
- operatorProjection, operatorProjectionOK = provider.OperatorStateProjection(operatorResourceID, time.Now())
- if operatorProjectionOK {
- f.ResourceCriticality = normalizeResourceCriticality(operatorProjection.Criticality)
- } else {
- f.ResourceCriticality = ""
- }
+ if operatorStateProviderPresent {
+ if operatorProjectionOK {
+ f.ResourceCriticality = normalizeResourceCriticality(operatorProjection.Criticality)
+ } else {
+ f.ResourceCriticality = ""
}
- } else {
+ } else if s.resourceOperatorStateProvider == nil {
f.ResourceCriticality = normalizeResourceCriticality(f.ResourceCriticality)
}
if exists {
diff --git a/internal/ai/findings_test.go b/internal/ai/findings_test.go
index cb2f2c6a6d..ea1606f8d8 100644
--- a/internal/ai/findings_test.go
+++ b/internal/ai/findings_test.go
@@ -1711,6 +1711,145 @@ func TestFinding_RemindAt_RoundTripsThroughJSON(t *testing.T) {
}
}
+func TestFindingsStore_AddUsesCurrentOperatorStateProvider(t *testing.T) {
+ for _, replace := range []bool{false, true} {
+ name := "remove"
+ if replace {
+ name = "replace"
+ }
+ t.Run(name, func(t *testing.T) {
+ store := NewFindingsStore()
+ entered, release := make(chan struct{}), make(chan struct{})
+ store.SetResourceOperatorStateProvider(ResourceOperatorStateProviderFunc(func(string, time.Time) (ResourceOperatorStateProjection, bool) {
+ close(entered)
+ <-release
+ return ResourceOperatorStateProjection{IntentionallyOffline: true, Criticality: "high"}, true
+ }))
+ addDone, changed := make(chan struct{}), make(chan struct{})
+ go func() {
+ store.Add(&Finding{ID: "f1", ResourceID: "vm:101", Title: "Unavailable", Severity: FindingSeverityInfo})
+ close(addDone)
+ }()
+ defer func() {
+ close(release)
+ select {
+ case <-changed:
+ case <-time.After(3 * time.Second):
+ t.Error("provider change did not finish")
+ }
+ select {
+ case <-addDone:
+ case <-time.After(3 * time.Second):
+ t.Error("finding add did not finish")
+ }
+ }()
+ select {
+ case <-entered:
+ case <-time.After(3 * time.Second):
+ t.Fatal("operator-state lookup did not start")
+ }
+ go func() {
+ var next ResourceOperatorStateProvider
+ if replace {
+ next = ResourceOperatorStateProviderFunc(func(string, time.Time) (ResourceOperatorStateProjection, bool) {
+ // A provider may read the store while projecting state.
+ store.GetSummary()
+ return ResourceOperatorStateProjection{Criticality: "low"}, true
+ })
+ }
+ store.SetResourceOperatorStateProvider(next)
+ close(changed)
+ }()
+ select {
+ case <-changed:
+ case <-time.After(3 * time.Second):
+ t.Error("provider cannot be replaced while its previous lookup is blocked")
+ return
+ }
+ // Release the old result, then wait for the new projection to commit.
+ release <- struct{}{}
+ select {
+ case <-addDone:
+ case <-time.After(3 * time.Second):
+ t.Error("provider re-entry blocked finding add")
+ return
+ }
+ got := store.Get("f1")
+ if got == nil || got.DismissedReason != "" {
+ t.Fatalf("stale provider suppressed the finding: %+v", got)
+ }
+ want := ""
+ if replace {
+ want = "low"
+ }
+ if got.ResourceCriticality != want {
+ t.Errorf("criticality = %q, want %q from current provider", got.ResourceCriticality, want)
+ }
+ })
+ }
+}
+
+func TestFindingsStore_AddRefreshesReboundOperatorResource(t *testing.T) {
+ store := NewFindingsStore()
+ store.Add(&Finding{ID: "f1", ResourceID: "vm:100", Title: "Unavailable", Severity: FindingSeverityInfo})
+ entered, release := make(chan struct{}), make(chan struct{})
+ store.SetResourceOperatorStateProvider(ResourceOperatorStateProviderFunc(func(id string, _ time.Time) (ResourceOperatorStateProjection, bool) {
+ if id == "vm:100" {
+ close(entered)
+ <-release
+ return ResourceOperatorStateProjection{IntentionallyOffline: true, Criticality: "high"}, true
+ }
+ return ResourceOperatorStateProjection{Criticality: "low"}, id == "vm:101"
+ }))
+ addDone, rebound := make(chan struct{}), make(chan error, 1)
+ go func() {
+ store.Add(&Finding{ID: "f1", Title: "Unavailable", Severity: FindingSeverityInfo})
+ close(addDone)
+ }()
+ defer func() {
+ close(release)
+ select {
+ case <-addDone:
+ case <-time.After(3 * time.Second):
+ t.Error("finding add did not finish")
+ return
+ }
+ if err := store.ForceSave(); err != nil {
+ t.Error(err)
+ }
+ }()
+ select {
+ case <-entered:
+ case <-time.After(3 * time.Second):
+ t.Fatal("operator-state lookup did not start")
+ }
+ go func() {
+ rebound <- store.SetPersistence(&recordingPersistence{findings: map[string]*Finding{
+ "f1": {ID: "f1", ResourceID: "vm:101", Title: "Unavailable", Severity: FindingSeverityInfo},
+ }})
+ }()
+ select {
+ case err := <-rebound:
+ if err != nil {
+ t.Fatal(err)
+ }
+ case <-time.After(3 * time.Second):
+ t.Error("retained identity cannot change during an external lookup")
+ return
+ }
+ release <- struct{}{}
+ select {
+ case <-addDone:
+ case <-time.After(3 * time.Second):
+ t.Error("finding add did not finish after identity changed")
+ return
+ }
+ got := store.Get("f1")
+ if got == nil || got.ResourceID != "vm:101" || got.ResourceCriticality != "low" || got.DismissedReason != "" {
+ t.Fatalf("finding committed a stale resource projection: %+v", got)
+ }
+}
+
func TestFindingsStore_Add_AutoAcknowledgesDuringMaintenanceWindow(t *testing.T) {
// When the operator has set a maintenance window covering the
// finding's resource, the new-finding path must auto-dismiss the
diff --git a/internal/ai/patrol_test.go b/internal/ai/patrol_test.go
index a84569227e..f8944eec88 100644
--- a/internal/ai/patrol_test.go
+++ b/internal/ai/patrol_test.go
@@ -7,6 +7,7 @@ import (
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/ai/memory"
+ "github.com/rcourtman/pulse-go-rewrite/internal/config"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
ur "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
)
@@ -1089,6 +1090,68 @@ func TestPatrolService_GetIntelligence(t *testing.T) {
}
}
+func TestPatrolService_StopWhileOperatorStateLookupBlocked(t *testing.T) {
+ ps := NewPatrolService(nil, nil)
+ persistence := config.NewConfigPersistence(t.TempDir())
+ if err := ps.SetFindingsPersistence(NewFindingsPersistenceAdapter(persistence)); err != nil {
+ t.Fatal(err)
+ }
+ ps.findings.Add(&Finding{ID: "retained", ResourceID: "vm:100", Title: "Retained finding", Severity: FindingSeverityWarning})
+ entered, release := make(chan struct{}), make(chan struct{})
+ ps.findings.SetResourceOperatorStateProvider(ResourceOperatorStateProviderFunc(func(string, time.Time) (ResourceOperatorStateProjection, bool) {
+ close(entered)
+ <-release
+ return ResourceOperatorStateProjection{}, false
+ }))
+ addDone, stopDone := make(chan struct{}), make(chan struct{})
+ go func() {
+ ps.findings.Add(&Finding{ID: "pending", ResourceID: "vm:101", Title: "Pending lookup", Severity: FindingSeverityInfo})
+ close(addDone)
+ }()
+ defer func() {
+ close(release)
+ select {
+ case <-addDone:
+ case <-time.After(3 * time.Second):
+ t.Error("finding add did not finish after its lookup was released")
+ return
+ }
+ select {
+ case <-stopDone:
+ case <-time.After(3 * time.Second):
+ t.Error("Patrol stop did not finish after its lookup was released")
+ return
+ }
+ if err := ps.findings.ForceSave(); err != nil {
+ t.Error(err)
+ }
+ }()
+ select {
+ case <-entered:
+ case <-time.After(3 * time.Second):
+ t.Fatal("operator-state lookup did not start")
+ }
+ ps.running = true
+ ps.stopCh = make(chan struct{})
+ go func() { ps.Stop(); close(stopDone) }()
+ select {
+ case <-stopDone:
+ case <-time.After(3 * time.Second):
+ t.Error("Patrol shutdown cannot save retained findings while an external operator-state lookup is blocked")
+ return
+ }
+ saved, err := persistence.LoadAIFindings()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if saved.Findings["retained"] == nil {
+ t.Error("shutdown did not preserve the retained finding")
+ }
+ if saved.Findings["pending"] != nil {
+ t.Error("shutdown saved a finding whose operator-state lookup had not completed")
+ }
+}
+
func TestPatrolService_Stop(t *testing.T) {
ps := NewPatrolService(nil, nil)