Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Keep a recovered Patrol run's own status
rcourtman0ff9b26fb14c
diff --git a/docs/release-control/v6/internal/subsystems/ai-runtime.md b/docs/release-control/v6/internal/subsystems/ai-runtime.mdindex 8d82183891..77aff3740e 100644--- a/docs/release-control/v6/internal/subsystems/ai-runtime.md+++ b/docs/release-control/v6/internal/subsystems/ai-runtime.md@@ -7440,9 +7440,11 @@ so a result or a newer run that landed after the sweep looked is kept. A runthat started before the finding's current occurrence is recorded only as`interrupted`, never with its session's result. An outcome the actionlifecycle recorded on the finding during the run, such as-`fix_queued`, is kept and the run completes, so a queued fix still blocks-another investigation. The close stamps `LastInvestigatedAt` like any other-closed run.+`fix_queued`, is kept, so a queued fix still blocks another investigation.+The run keeps the status it closed with: a run that a restart or its limit+cut short after it planned a fix is `failed`, as the orchestrator records a+run that fails after planning, not `completed`. The close stamps+`LastInvestigatedAt` like any other closed run.Two outcomes join the vocabulary. `unclassified` records a completed run thatleft neither a typed conclusion nor a recognisable marker. The privatediff --git a/internal/ai/findings.go b/internal/ai/findings.goindex b644eb0f03..9b719ab5f8 100644--- a/internal/ai/findings.go+++ b/internal/ai/findings.go@@ -2786,9 +2786,10 @@ func (s *FindingsStore) updateInvestigationLocked(f *Finding, sessionID, status,// when the orchestrator holds none. A run that started before the finding's// current occurrence is only recorded as interrupted: its result is not this// occurrence's. A run starts with no outcome, so an outcome already present-// came from the action lifecycle during this run; it is kept, and the run-// completes, so a queued fix still blocks another investigation. The close-// becomes the finding's latest investigation time.+// came from the action lifecycle during this run; it is kept, so a queued fix+// still blocks another investigation. The run still closes with its own status:+// one cut short after it planned a fix failed, as it reads when it closes+// itself. The close becomes the finding's latest investigation time.func (s *FindingsStore) RecoverStuckInvestigation(id, sessionID string, runStartedAt time.Time, status InvestigationStatus, outcome InvestigationOutcome, at time.Time) bool {s.mu.Lock()f, exists := s.findings[id]@@ -2801,9 +2802,7 @@ func (s *FindingsStore) RecoverStuckInvestigation(id, sessionID string, runStart}prevStatus := f.InvestigationStatusprevOutcome := f.InvestigationOutcome- if prevOutcome != "" {- status = InvestigationStatusCompleted- } else {+ if prevOutcome == "" {f.InvestigationOutcome = string(outcome)if outcome == InvestigationOutcomeFixVerified {s.resolveVerifiedFindingLocked(f, prevOutcome)diff --git a/internal/ai/findings_test.go b/internal/ai/findings_test.goindex ad8b397150..cb2f2c6a6d 100644--- a/internal/ai/findings_test.go+++ b/internal/ai/findings_test.go@@ -2672,11 +2672,12 @@ func TestCompleteInvestigationClosesARunFromAnEarlierOccurrenceAsTheSweepWould(t}assertInvestigationState(t, findings, "settled", InvestigationStatusFailed, InvestigationOutcomeInterrupted)// The close keeps an outcome the action lifecycle recorded, as the sweep- // does, so the queued fix still blocks another investigation.+ // does, so the queued fix still blocks another investigation, and the run+ // still reads as failed.if complete("queued", beforeRegression) {t.Fatal("a run from before the regression recorded its result")}- assertInvestigationState(t, findings, "queued", InvestigationStatusCompleted, InvestigationOutcomeFixQueued)+ assertInvestigationState(t, findings, "queued", InvestigationStatusFailed, InvestigationOutcomeFixQueued)if findings.Get("queued").ShouldInvestigate("approval") {t.Fatal("a queued fix must keep blocking another investigation")}diff --git a/internal/ai/patrol_investigation_finalize_test.go b/internal/ai/patrol_investigation_finalize_test.goindex 1c5c409847..25a73dcad6 100644--- a/internal/ai/patrol_investigation_finalize_test.go+++ b/internal/ai/patrol_investigation_finalize_test.go@@ -169,11 +169,11 @@ func TestFinalizeInvestigationRunLeavesActionOutcomePushesToReconciliation(t *te}}-// The stuck-run sweep finalizes each run it recovers: one whose session closed-// without its close reaching the finding, such as a run a restart interrupted-// after it planned a fix, and one still going past its limit, as failed. The-// orchestrator's own session for a run past its limit is left as it is, so the-// run's real close can still land and finalize it again.+// The stuck-run sweep finalizes each run it recovers, both as failed: one whose+// session closed without its close reaching the finding, such as a run a+// restart interrupted after it planned a fix, and one still going past its+// limit. The orchestrator's own session for a run past its limit is left as it+// is, so the run's real close can still land and finalize it again.func TestRecoverStuckInvestigationsFinalizesTheRunsItRecovers(t *testing.T) {now := time.Now()closedAt := now.Add(-time.Second)@@ -198,7 +198,7 @@ func TestRecoverStuckInvestigationsFinalizesTheRunsItRecovers(t *testing.T) {patrol.recoverStuckInvestigations()- assertInvestigationState(t, findings, "restarted", InvestigationStatusCompleted, InvestigationOutcomeFixQueued)+ assertInvestigationState(t, findings, "restarted", InvestigationStatusFailed, InvestigationOutcomeFixQueued)assertInvestigationState(t, findings, "hung", InvestigationStatusFailed, InvestigationOutcomeTimedOut)if len(closed) != 2 {t.Fatalf("closed runs = %v, want both recovered runs", closed)diff --git a/internal/ai/patrol_investigation_limits_test.go b/internal/ai/patrol_investigation_limits_test.goindex 205e20b95e..82f326b55c 100644--- a/internal/ai/patrol_investigation_limits_test.go+++ b/internal/ai/patrol_investigation_limits_test.go@@ -144,7 +144,7 @@ func TestRecoverStuckInvestigationsKeepsAnActionOutcome(t *testing.T) {now := time.Now()closedAWhileAgo := now.Add(-investigationRunGrace - time.Minute)findings := NewFindingsStore()- for _, id := range []string{"queued", "queued-interrupted"} {+ for _, id := range []string{"queued", "queued-interrupted", "queued-closed"} {f := runningInvestigationFinding(id, "chat-"+id, nil)// The action lifecycle recorded the queued fix while the run was// still running, and a restart came before the run finished.@@ -154,12 +154,20 @@ func TestRecoverStuckInvestigationsKeepsAnActionOutcome(t *testing.T) {orchestrator := &sessionReportingOrchestrator{sessions: map[string]*InvestigationSession{"queued": {ID: "inv-queued", FindingID: "queued", SessionID: "chat-queued", Status: aicontracts.InvestigationStatusFailed, Outcome: aicontracts.OutcomeFixQueued, StartedAt: now.Add(-time.Hour), CompletedAt: &closedAWhileAgo},"queued-interrupted": {ID: "inv-queued-interrupted", FindingID: "queued-interrupted", SessionID: "chat-queued-interrupted", Status: aicontracts.InvestigationStatusFailed, Outcome: aicontracts.OutcomeInterrupted, StartedAt: now.Add(-time.Hour), CompletedAt: &closedAWhileAgo},+ // This run closed itself, but its update never reached the finding.+ "queued-closed": {ID: "inv-queued-closed", FindingID: "queued-closed", SessionID: "chat-queued-closed", Status: aicontracts.InvestigationStatusCompleted, Outcome: aicontracts.OutcomeFixQueued, StartedAt: now.Add(-time.Hour), CompletedAt: &closedAWhileAgo},}}patrol := &PatrolService{findings: findings, investigationOrchestrator: orchestrator}patrol.recoverStuckInvestigations()- for _, id := range []string{"queued", "queued-interrupted"} {- assertInvestigationState(t, findings, id, InvestigationStatusCompleted, InvestigationOutcomeFixQueued)+ // Each run keeps the status it closed with: a run cut short after it+ // planned the fix failed, as it reads when such a run closes itself.+ for id, status := range map[string]InvestigationStatus{+ "queued": InvestigationStatusFailed,+ "queued-interrupted": InvestigationStatusFailed,+ "queued-closed": InvestigationStatusCompleted,+ } {+ assertInvestigationState(t, findings, id, status, InvestigationOutcomeFixQueued)if findings.Get(id).ShouldInvestigate("approval") {t.Fatalf("%s: a queued fix must keep blocking another investigation", id)}