Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Keep a recovered Patrol run's own status

rcourtman0ff9b26fb14c
diff --git a/docs/release-control/v6/internal/subsystems/ai-runtime.md b/docs/release-control/v6/internal/subsystems/ai-runtime.md
index 8d82183891..77aff3740e 100644
--- a/docs/release-control/v6/internal/subsystems/ai-runtime.md
+++ b/docs/release-control/v6/internal/subsystems/ai-runtime.md
@@ -7440,9 +7440,11 @@ so a result or a newer run that landed after the sweep looked is kept. A run
that started before the finding's current occurrence is recorded only as
`interrupted`, never with its session's result. An outcome the action
lifecycle recorded on the finding during the run, such as
-`fix_queued`, is kept and the run completes, so a queued fix still blocks
-another investigation. The close stamps `LastInvestigatedAt` like any other
-closed run.
+`fix_queued`, is kept, so a queued fix still blocks another investigation.
+The run keeps the status it closed with: a run that a restart or its limit
+cut short after it planned a fix is `failed`, as the orchestrator records a
+run that fails after planning, not `completed`. The close stamps
+`LastInvestigatedAt` like any other closed run.
Two outcomes join the vocabulary. `unclassified` records a completed run that
left neither a typed conclusion nor a recognisable marker. The private
diff --git a/internal/ai/findings.go b/internal/ai/findings.go
index b644eb0f03..9b719ab5f8 100644
--- a/internal/ai/findings.go
+++ b/internal/ai/findings.go
@@ -2786,9 +2786,10 @@ func (s *FindingsStore) updateInvestigationLocked(f *Finding, sessionID, status,
// when the orchestrator holds none. A run that started before the finding's
// current occurrence is only recorded as interrupted: its result is not this
// occurrence's. A run starts with no outcome, so an outcome already present
-// came from the action lifecycle during this run; it is kept, and the run
-// completes, so a queued fix still blocks another investigation. The close
-// becomes the finding's latest investigation time.
+// came from the action lifecycle during this run; it is kept, so a queued fix
+// still blocks another investigation. The run still closes with its own status:
+// one cut short after it planned a fix failed, as it reads when it closes
+// itself. The close becomes the finding's latest investigation time.
func (s *FindingsStore) RecoverStuckInvestigation(id, sessionID string, runStartedAt time.Time, status InvestigationStatus, outcome InvestigationOutcome, at time.Time) bool {
s.mu.Lock()
f, exists := s.findings[id]
@@ -2801,9 +2802,7 @@ func (s *FindingsStore) RecoverStuckInvestigation(id, sessionID string, runStart
}
prevStatus := f.InvestigationStatus
prevOutcome := f.InvestigationOutcome
- if prevOutcome != "" {
- status = InvestigationStatusCompleted
- } else {
+ if prevOutcome == "" {
f.InvestigationOutcome = string(outcome)
if outcome == InvestigationOutcomeFixVerified {
s.resolveVerifiedFindingLocked(f, prevOutcome)
diff --git a/internal/ai/findings_test.go b/internal/ai/findings_test.go
index ad8b397150..cb2f2c6a6d 100644
--- a/internal/ai/findings_test.go
+++ b/internal/ai/findings_test.go
@@ -2672,11 +2672,12 @@ func TestCompleteInvestigationClosesARunFromAnEarlierOccurrenceAsTheSweepWould(t
}
assertInvestigationState(t, findings, "settled", InvestigationStatusFailed, InvestigationOutcomeInterrupted)
// The close keeps an outcome the action lifecycle recorded, as the sweep
- // does, so the queued fix still blocks another investigation.
+ // does, so the queued fix still blocks another investigation, and the run
+ // still reads as failed.
if complete("queued", beforeRegression) {
t.Fatal("a run from before the regression recorded its result")
}
- assertInvestigationState(t, findings, "queued", InvestigationStatusCompleted, InvestigationOutcomeFixQueued)
+ assertInvestigationState(t, findings, "queued", InvestigationStatusFailed, InvestigationOutcomeFixQueued)
if findings.Get("queued").ShouldInvestigate("approval") {
t.Fatal("a queued fix must keep blocking another investigation")
}
diff --git a/internal/ai/patrol_investigation_finalize_test.go b/internal/ai/patrol_investigation_finalize_test.go
index 1c5c409847..25a73dcad6 100644
--- a/internal/ai/patrol_investigation_finalize_test.go
+++ b/internal/ai/patrol_investigation_finalize_test.go
@@ -169,11 +169,11 @@ func TestFinalizeInvestigationRunLeavesActionOutcomePushesToReconciliation(t *te
}
}
-// The stuck-run sweep finalizes each run it recovers: one whose session closed
-// without its close reaching the finding, such as a run a restart interrupted
-// after it planned a fix, and one still going past its limit, as failed. The
-// orchestrator's own session for a run past its limit is left as it is, so the
-// run's real close can still land and finalize it again.
+// The stuck-run sweep finalizes each run it recovers, both as failed: one whose
+// session closed without its close reaching the finding, such as a run a
+// restart interrupted after it planned a fix, and one still going past its
+// limit. The orchestrator's own session for a run past its limit is left as it
+// is, so the run's real close can still land and finalize it again.
func TestRecoverStuckInvestigationsFinalizesTheRunsItRecovers(t *testing.T) {
now := time.Now()
closedAt := now.Add(-time.Second)
@@ -198,7 +198,7 @@ func TestRecoverStuckInvestigationsFinalizesTheRunsItRecovers(t *testing.T) {
patrol.recoverStuckInvestigations()
- assertInvestigationState(t, findings, "restarted", InvestigationStatusCompleted, InvestigationOutcomeFixQueued)
+ assertInvestigationState(t, findings, "restarted", InvestigationStatusFailed, InvestigationOutcomeFixQueued)
assertInvestigationState(t, findings, "hung", InvestigationStatusFailed, InvestigationOutcomeTimedOut)
if len(closed) != 2 {
t.Fatalf("closed runs = %v, want both recovered runs", closed)
diff --git a/internal/ai/patrol_investigation_limits_test.go b/internal/ai/patrol_investigation_limits_test.go
index 205e20b95e..82f326b55c 100644
--- a/internal/ai/patrol_investigation_limits_test.go
+++ b/internal/ai/patrol_investigation_limits_test.go
@@ -144,7 +144,7 @@ func TestRecoverStuckInvestigationsKeepsAnActionOutcome(t *testing.T) {
now := time.Now()
closedAWhileAgo := now.Add(-investigationRunGrace - time.Minute)
findings := NewFindingsStore()
- for _, id := range []string{"queued", "queued-interrupted"} {
+ for _, id := range []string{"queued", "queued-interrupted", "queued-closed"} {
f := runningInvestigationFinding(id, "chat-"+id, nil)
// The action lifecycle recorded the queued fix while the run was
// still running, and a restart came before the run finished.
@@ -154,12 +154,20 @@ func TestRecoverStuckInvestigationsKeepsAnActionOutcome(t *testing.T) {
orchestrator := &sessionReportingOrchestrator{sessions: map[string]*InvestigationSession{
"queued": {ID: "inv-queued", FindingID: "queued", SessionID: "chat-queued", Status: aicontracts.InvestigationStatusFailed, Outcome: aicontracts.OutcomeFixQueued, StartedAt: now.Add(-time.Hour), CompletedAt: &closedAWhileAgo},
"queued-interrupted": {ID: "inv-queued-interrupted", FindingID: "queued-interrupted", SessionID: "chat-queued-interrupted", Status: aicontracts.InvestigationStatusFailed, Outcome: aicontracts.OutcomeInterrupted, StartedAt: now.Add(-time.Hour), CompletedAt: &closedAWhileAgo},
+ // This run closed itself, but its update never reached the finding.
+ "queued-closed": {ID: "inv-queued-closed", FindingID: "queued-closed", SessionID: "chat-queued-closed", Status: aicontracts.InvestigationStatusCompleted, Outcome: aicontracts.OutcomeFixQueued, StartedAt: now.Add(-time.Hour), CompletedAt: &closedAWhileAgo},
}}
patrol := &PatrolService{findings: findings, investigationOrchestrator: orchestrator}
patrol.recoverStuckInvestigations()
- for _, id := range []string{"queued", "queued-interrupted"} {
- assertInvestigationState(t, findings, id, InvestigationStatusCompleted, InvestigationOutcomeFixQueued)
+ // Each run keeps the status it closed with: a run cut short after it
+ // planned the fix failed, as it reads when such a run closes itself.
+ for id, status := range map[string]InvestigationStatus{
+ "queued": InvestigationStatusFailed,
+ "queued-interrupted": InvestigationStatusFailed,
+ "queued-closed": InvestigationStatusCompleted,
+ } {
+ assertInvestigationState(t, findings, id, status, InvestigationOutcomeFixQueued)
if findings.Get(id).ShouldInvestigate("approval") {
t.Fatalf("%s: a queued fix must keep blocking another investigation", id)
}