Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
fix: retain scheduled follow-up after incomplete investigations
rcourtman51a45493d11b
diff --git a/docs/release-control/v6/internal/subsystems/ai-runtime.md b/docs/release-control/v6/internal/subsystems/ai-runtime.mdindex 16cc08de1b..0a75d2a053 100644--- a/docs/release-control/v6/internal/subsystems/ai-runtime.md+++ b/docs/release-control/v6/internal/subsystems/ai-runtime.md@@ -8079,6 +8079,28 @@ inspection failure, stale facts, or mismatched identities remaincontinues to project only `agent_attested` evidence, so adding the productionobserver does not loosen the verification rule.+### Incomplete investigation follow-up++An empty or unusable final model response, provider error, or final-summary+timeout does not constitute a model conclusion. The shared agentic loop retains+a readable failure receipt and returns the original error identity for Patrol+investigations. Interactive chat retains its existing best-effort fallback.+The investigation service must not advance a captured proposal after that error.++The private orchestrator clears the previous finding outcome when starting a+new session, after retaining its prior context. Failed runs preserve evidence,+counts and any existing action reference. They remain failed rather than being+classified from fallback prose as a completed `needs_attention` investigation.+Failed runs without a conclusion, including timeouts, may be assessed at the+next scheduled Watch without another cooldown. Existing attempt limits,+admission, autonomy and action authority still apply. A real completed model+decision requiring human attention remains terminal.++Proofs: `internal/ai/chat/agentic_final_failure_test.go`,+`internal/ai/findings_provider_failure_test.go`, and the private+`TestProviderFailureClearsPriorConclusionAndRetainsEvidence` regression.+These causal tests do not themselves qualify model-owned unattended recovery.+### Proxmox guest lifecycle Patrol detector floor`internal/ai/findings_proxmox_lifecycle.go` owns the deterministic productiondiff --git a/internal/ai/chat/agentic.go b/internal/ai/chat/agentic.goindex bdfa617b14..961e9e2537 100644--- a/internal/ai/chat/agentic.go+++ b/internal/ai/chat/agentic.go@@ -1464,8 +1464,7 @@ func (a *AgenticLoop) executeWithTools(ctx context.Context, sessionID string, me}log.Debug().Msg("agentic loop complete - no tool calls")- resultMessages = a.ensureFinalTextResponse(ctx, sessionID, resultMessages, providerMessages, callback)- return resultMessages, nil+ return a.ensureFinalTextResponse(ctx, sessionID, resultMessages, providerMessages, callback)}// === Execute tool calls (three-phase pipeline) ===@@ -1954,6 +1953,5 @@ func (a *AgenticLoop) executeWithTools(ctx context.Context, sessionID string, me}log.Warn().Int("max_turns", maxTurns).Str("session_id", sessionID).Msg("agentic loop hit max turns limit")- resultMessages = a.ensureFinalTextResponse(ctx, sessionID, resultMessages, providerMessages, callback)- return resultMessages, nil+ return a.ensureFinalTextResponse(ctx, sessionID, resultMessages, providerMessages, callback)}diff --git a/internal/ai/chat/agentic_additional_test.go b/internal/ai/chat/agentic_additional_test.goindex 2ad8290c6c..ce8f48ddcd 100644--- a/internal/ai/chat/agentic_additional_test.go+++ b/internal/ai/chat/agentic_additional_test.go@@ -1378,13 +1378,16 @@ func TestEnsureFinalTextResponse(t *testing.T) {provider := &stubStreamingProvider{}loop := &AgenticLoop{provider: provider, baseSystemPrompt: "prompt"}- result := loop.ensureFinalTextResponse(+ result, summaryErr := loop.ensureFinalTextResponse(context.Background(),"session-1",[]Message{{Role: "assistant", Content: ""}},[]providers.Message{{Role: "assistant", Content: ""}},func(event StreamEvent) {},)+ if summaryErr != nil {+ t.Fatal(summaryErr)+ }if len(result) != 2 {t.Fatalf("expected summary message to be appended")}@@ -1398,7 +1401,7 @@ func TestEnsureFinalTextResponse(t *testing.T) {provider.chatStream = func(ctx context.Context, req providers.ChatRequest, callback providers.StreamCallback) error {return errors.New("boom")}- result2 := loop.ensureFinalTextResponse(+ result2, summaryErr2 := loop.ensureFinalTextResponse(context.Background(),"session-2",[]Message{@@ -1408,6 +1411,9 @@ func TestEnsureFinalTextResponse(t *testing.T) {[]providers.Message{{Role: "assistant", Content: ""}},func(event StreamEvent) {},)+ if summaryErr2 != nil {+ t.Fatal(summaryErr2)+ }if len(result2) != 3 {t.Fatalf("expected fallback summary message when provider errors")}@@ -1422,7 +1428,7 @@ func TestEnsureFinalTextResponse(t *testing.T) {callback(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{InputTokens: 1, OutputTokens: 1}})return nil}- result3 := loop.ensureFinalTextResponse(+ result3, summaryErr3 := loop.ensureFinalTextResponse(context.Background(),"session-3",[]Message{@@ -1432,6 +1438,9 @@ func TestEnsureFinalTextResponse(t *testing.T) {[]providers.Message{{Role: "assistant", Content: ""}},func(event StreamEvent) {},)+ if summaryErr3 != nil {+ t.Fatal(summaryErr3)+ }if len(result3) != 3 {t.Fatalf("expected fallback summary when provider returns empty content")}@@ -1453,7 +1462,7 @@ func TestEnsureFinalTextResponseRequiresAssistantTextAfterLatestToolResult(t *teloop := &AgenticLoop{provider: provider, baseSystemPrompt: "prompt"}var emitted strings.Builder- result := loop.ensureFinalTextResponse(+ result, summaryErr := loop.ensureFinalTextResponse(context.Background(),"session-pre-tool-preamble",[]Message{@@ -1494,6 +1503,9 @@ func TestEnsureFinalTextResponseRequiresAssistantTextAfterLatestToolResult(t *te},)+ if summaryErr != nil {+ t.Fatal(summaryErr)+ }if len(result) != 5 {t.Fatalf("expected final summary to be appended after tool result, got %d messages", len(result))}diff --git a/internal/ai/chat/agentic_final.go b/internal/ai/chat/agentic_final.goindex 8e1119b753..15a66c7593 100644--- a/internal/ai/chat/agentic_final.go+++ b/internal/ai/chat/agentic_final.go@@ -29,7 +29,7 @@ func (a *AgenticLoop) ensureFinalTextResponse(resultMessages []Message,providerMessages []providers.Message,callback StreamCallback,-) []Message {+) ([]Message, error) {return a.ensureFinalTextResponseWithSystemPrompt(ctx, sessionID, resultMessages, providerMessages, callback, "")}@@ -43,9 +43,9 @@ func (a *AgenticLoop) ensureFinalTextResponseWithSystemPrompt(providerMessages []providers.Message,callback StreamCallback,systemPromptOverride string,-) []Message {+) ([]Message, error) {if hasFinalAssistantText(resultMessages) {- return resultMessages+ return resultMessages, nil}// No text content from the model. Make a final text-only call.@@ -136,16 +136,17 @@ func (a *AgenticLoop) ensureFinalTextResponseWithSystemPrompt(}}- if summaryErr == nil && summaryBuilder.Len() > 0 {+ summaryText := cleanToolCallArtifacts(summaryBuilder.String())+ if summaryErr == nil && !suppressLeakedToolContent && strings.TrimSpace(summaryText) != "" {summaryMsg := Message{ID: uuid.New().String(),Role: "assistant",- Content: cleanToolCallArtifacts(summaryBuilder.String()),+ Content: summaryText,Timestamp: time.Now(),}resultMessages = append(resultMessages, summaryMsg)log.Info().Str("session_id", sessionID).Int("summary_len", summaryBuilder.Len()).Msg("[AgenticLoop] Final summary produced")- return resultMessages+ return resultMessages, nil}if summaryErr != nil {@@ -157,6 +158,9 @@ func (a *AgenticLoop) ensureFinalTextResponseWithSystemPrompt(// Deterministic fallback so the user always gets a usable response even if// the model fails to emit final text.fallback := buildAutomaticFallbackSummary(resultMessages)+ if isPatrolInvestigationExecution(a.currentExecutionProfile()) {+ fallback = "The model did not return a final narrative for this investigation. Evidence remains in the investigation history. Patrol can reassess this finding on a later scheduled check within the current policy and attempt limit."+ }fallbackMsg := Message{ID: uuid.New().String(),Role: "assistant",@@ -168,7 +172,18 @@ func (a *AgenticLoop) ensureFinalTextResponseWithSystemPrompt(callback(StreamEvent{Type: "content", Data: jsonData})log.Warn().Str("session_id", sessionID).Int("summary_len", len(fallback)).Msg("[AgenticLoop] Emitted deterministic fallback summary")- return resultMessages+ // A readable receipt is not a completed model investigation. Preserve it+ // for the operator, but keep failure on the error channel so the caller+ // cannot classify fallback prose as a request for human intervention or+ // advance a captured action proposal. Interactive chat keeps its existing+ // best-effort narrative behaviour.+ if isPatrolInvestigationExecution(a.currentExecutionProfile()) {+ if summaryErr != nil {+ return resultMessages, fmt.Errorf("investigation final response failed: %w", summaryErr)+ }+ return resultMessages, fmt.Errorf("investigation final response was empty or unusable")+ }+ return resultMessages, nil}func hasFinalAssistantText(resultMessages []Message) bool {diff --git a/internal/ai/chat/agentic_final_failure_test.go b/internal/ai/chat/agentic_final_failure_test.gonew file mode 100644index 0000000000..60d981715d--- /dev/null+++ b/internal/ai/chat/agentic_final_failure_test.go@@ -0,0 +1,104 @@+package chat++import (+ "context"+ "errors"+ "strings"+ "testing"+ "time"++ "github.com/rcourtman/pulse-go-rewrite/internal/ai/providers"+ "github.com/rcourtman/pulse-go-rewrite/internal/ai/tools"+)++func TestInvestigationFinalResponseFailureRetainsReceiptAndError(t *testing.T) {+ for _, tc := range []struct {+ name string+ err error+ }{+ {"empty", nil}, {"timeout", context.DeadlineExceeded}, {"provider", errors.New("provider unavailable")},+ } {+ t.Run(tc.name, func(t *testing.T) {+ calls := 0+ provider := &stubStreamingProvider{chatStream: func(ctx context.Context, req providers.ChatRequest, callback providers.StreamCallback) error {+ calls+++ if calls == 1 {+ callback(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{StopReason: "tool_use"}})+ return nil+ }+ if len(req.Tools) != 0 {+ t.Fatal("summary offered write tools")+ }+ return tc.err+ }}+ loop := NewAgenticLoop(provider, tools.NewPulseToolExecutor(tools.ExecutorConfig{}), "Investigate current evidence")+ loop.SetExecutionProfile(tools.ProfilePatrolInvestigation)+ messages, err := loop.ExecuteWithTools(context.Background(), "incomplete", []Message{{Role: "user", Content: "Check service"}}, nil, func(StreamEvent) {})+ if err == nil {+ t.Fatal("incomplete provider run was reported as successful")+ }+ if tc.err != nil && !errors.Is(err, tc.err) {+ t.Fatalf("lost error identity: %v", err)+ }+ if calls != 2 {+ t.Fatalf("provider calls=%d, want original plus bounded summary", calls)+ }+ if len(messages) == 0 || !strings.Contains(messages[len(messages)-1].Content, "final narrative") {+ t.Fatalf("missing retained fallback receipt: %#v", messages)+ }+ })+ }+}++func TestInvestigationEmptyResponseCanRecoverWithRealModelConclusion(t *testing.T) {+ calls := 0+ p := &stubStreamingProvider{chatStream: func(ctx context.Context, req providers.ChatRequest, callback providers.StreamCallback) error {+ calls+++ if calls == 2 {+ callback(providers.StreamEvent{Type: "content", Data: providers.ContentEvent{Text: "### Conclusion\nNEEDS_ATTENTION: upstream owner must repair the dependency"}})+ }+ callback(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{}})+ return nil+ }}+ loop := NewAgenticLoop(p, tools.NewPulseToolExecutor(tools.ExecutorConfig{}), "Investigate")+ loop.SetExecutionProfile(tools.ProfilePatrolInvestigation)+ messages, err := loop.ExecuteWithTools(context.Background(), "conclusion", []Message{{Role: "user", Content: "Check"}}, nil, func(StreamEvent) {})+ if err != nil || !strings.Contains(messages[len(messages)-1].Content, "NEEDS_ATTENTION:") {+ t.Fatalf("real model conclusion was discarded: %v %#v", err, messages)+ }+}++func TestInvestigationSummaryRetainsCallerDeadline(t *testing.T) {+ calls := 0+ p := &stubStreamingProvider{chatStream: func(ctx context.Context, req providers.ChatRequest, cb providers.StreamCallback) error {+ calls+++ if calls == 1 {+ cb(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{}})+ return nil+ }+ <-ctx.Done()+ return ctx.Err()+ }}+ loop := NewAgenticLoop(p, tools.NewPulseToolExecutor(tools.ExecutorConfig{}), "Investigate")+ loop.SetExecutionProfile(tools.ProfilePatrolInvestigation)+ ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)+ defer cancel()+ _, err := loop.ExecuteWithTools(ctx, "deadline", []Message{{Role: "user", Content: "Check"}}, nil, func(StreamEvent) {})+ if !errors.Is(err, context.DeadlineExceeded) {+ t.Fatalf("deadline identity lost: %v", err)+ }+}++func TestInvestigationWhitespaceSummaryIsNotAConclusion(t *testing.T) {+ p := &stubStreamingProvider{chatStream: func(ctx context.Context, req providers.ChatRequest, cb providers.StreamCallback) error {+ cb(providers.StreamEvent{Type: "content", Data: providers.ContentEvent{Text: " \n "}})+ cb(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{}})+ return nil+ }}+ loop := NewAgenticLoop(p, tools.NewPulseToolExecutor(tools.ExecutorConfig{}), "Investigate")+ loop.SetExecutionProfile(tools.ProfilePatrolInvestigation)+ _, err := loop.ExecuteWithTools(context.Background(), "whitespace", []Message{{Role: "user", Content: "Check"}}, nil, func(StreamEvent) {})+ if err == nil {+ t.Fatal("whitespace summary became a completed investigation")+ }+}diff --git a/internal/ai/findings.go b/internal/ai/findings.goindex 15d32e60ab..1c0fefb1cb 100644--- a/internal/ai/findings.go+++ b/internal/ai/findings.go@@ -464,6 +464,17 @@ func investigationCooldownForOutcome(outcome string) time.Duration {}}+func (f *Finding) investigationCooldown() time.Duration {+ if InvestigationStatus(f.InvestigationStatus) == InvestigationStatusFailed &&+ (f.InvestigationOutcome == "" || InvestigationOutcome(f.InvestigationOutcome) == InvestigationOutcomeTimedOut) {+ // The run failed before producing a conclusion. The scheduled Watch+ // cadence owns the next assessment; there is no additional cooldown.+ // Admission, attempt limits and action authority still apply.+ return 0+ }+ return investigationCooldownForOutcome(f.InvestigationOutcome)+}+func deriveLoopState(f *Finding) FindingLoopState {if f == nil {return FindingLoopStateDetected@@ -575,7 +586,7 @@ func (f *Finding) ShouldInvestigate(autonomyLevel string) bool {// Don't re-investigate within cooldown period// Timeout failures use a shorter cooldown (10min) since they're transientif f.LastInvestigatedAt != nil {- cooldown := investigationCooldownForOutcome(f.InvestigationOutcome)+ cooldown := f.investigationCooldown()if time.Since(*f.LastInvestigatedAt) < cooldown {return false}@@ -677,7 +688,7 @@ func (f *Finding) CanRetryInvestigation() bool {// Can't retry if still in cooldown// Timeout failures use a shorter cooldown (10min) since they're transientif f.LastInvestigatedAt != nil {- cooldown := investigationCooldownForOutcome(f.InvestigationOutcome)+ cooldown := f.investigationCooldown()if time.Since(*f.LastInvestigatedAt) < cooldown {return false}diff --git a/internal/ai/findings_provider_failure_test.go b/internal/ai/findings_provider_failure_test.gonew file mode 100644index 0000000000..3530ad4e62--- /dev/null+++ b/internal/ai/findings_provider_failure_test.go@@ -0,0 +1,46 @@+package ai++import (+ "github.com/rcourtman/pulse-go-rewrite/internal/config"+ "testing"+ "time"+)++func TestProviderFailureFollowupSurvivesRestart(t *testing.T) {+ for _, outcome := range []string{"", string(InvestigationOutcomeTimedOut)} {+ t.Run("outcome_"+outcome, func(t *testing.T) {+ persistence := config.NewConfigPersistence(t.TempDir())+ store := NewFindingsStore()+ if err := store.SetPersistence(NewFindingsPersistenceAdapter(persistence)); err != nil {+ t.Fatal(err)+ }+ now := time.Now()+ store.Add(&Finding{ID: "failed-provider", ResourceID: "app-container:service", ResourceType: "app-container", Title: "Service unavailable", Severity: FindingSeverityWarning, Category: FindingCategoryReliability, InvestigationSessionID: "failed-run", InvestigationStatus: string(InvestigationStatusFailed), InvestigationOutcome: outcome, LastInvestigatedAt: &now, InvestigationAttempts: 1})+ if err := store.ForceSave(); err != nil {+ t.Fatal(err)+ }+ restored := NewFindingsStore()+ if err := restored.SetPersistence(NewFindingsPersistenceAdapter(persistence)); err != nil {+ t.Fatal(err)+ }+ f := restored.Get("failed-provider")+ if !f.ShouldInvestigate("full") || !f.CanRetryInvestigation() {+ t.Fatal("persisted provider failure delayed the next scheduled model assessment")+ }+ f.InvestigationAttempts = maxInvestigationAttempts+ if f.ShouldInvestigate("full") {+ t.Fatal("bypassed attempt limit")+ }+ f.InvestigationAttempts = 1+ f.InvestigationStatus = string(InvestigationStatusRunning)+ if f.ShouldInvestigate("full") {+ t.Fatal("duplicated running investigation")+ }+ f.InvestigationStatus = string(InvestigationStatusCompleted)+ f.InvestigationOutcome = string(InvestigationOutcomeNeedsAttention)+ if f.ShouldInvestigate("full") {+ t.Fatal("overrode real human-attention conclusion")+ }+ })+ }+}diff --git a/internal/ai/findings_timeout_test.go b/internal/ai/findings_timeout_test.goindex 6c9f3aff28..e0d70c2165 100644--- a/internal/ai/findings_timeout_test.go+++ b/internal/ai/findings_timeout_test.go@@ -5,9 +5,8 @@ import ("time")-func TestShouldInvestigate_TimeoutUsesShortCooldown(t *testing.T) {- // Finding with timed_out outcome and LastInvestigatedAt 15 minutes ago- // should be eligible for re-investigation (shorter 10min cooldown)+func TestShouldInvestigate_FailedTimeoutUsesScheduledCadence(t *testing.T) {+ // Failed timeouts use the scheduled assessment cadence.fifteenMinsAgo := time.Now().Add(-15 * time.Minute)f := &Finding{ID: "f1",@@ -24,18 +23,18 @@ func TestShouldInvestigate_TimeoutUsesShortCooldown(t *testing.T) {t.Fatalf("expected timed-out finding (15min old) to be eligible for re-investigation")}- // Same finding but only 5 minutes ago — still within the 10min timeout cooldown+ // A failed run follows the Watch cadence without an additional cooldown.fiveMinsAgo := time.Now().Add(-5 * time.Minute)f.LastInvestigatedAt = &fiveMinsAgo- if f.ShouldInvestigate("approval") {- t.Fatalf("expected timed-out finding (5min old) to still be in cooldown")+ if !f.ShouldInvestigate("approval") {+ t.Fatalf("failed timeout should be eligible at the next scheduled assessment")}}-func TestShouldInvestigate_NonTimeoutKeepsLongCooldown(t *testing.T) {+func TestShouldInvestigate_FailedProviderUsesScheduledCadence(t *testing.T) {// Finding with no special outcome (regular failure) at 15 minutes ago- // should NOT be eligible — still within the 1-hour cooldown+ // remains eligible for the next scheduled assessment.fifteenMinsAgo := time.Now().Add(-15 * time.Minute)f := &Finding{ID: "f1",@@ -48,8 +47,8 @@ func TestShouldInvestigate_NonTimeoutKeepsLongCooldown(t *testing.T) {InvestigationAttempts: 1,}- if f.ShouldInvestigate("approval") {- t.Fatalf("expected non-timeout failure (15min old) to still be in 1hr cooldown")+ if !f.ShouldInvestigate("approval") {+ t.Fatalf("failed provider run should not wait another hour")}// After 2 hours, should be eligible