Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

fix: retain scheduled follow-up after incomplete investigations

rcourtman51a45493d11b
diff --git a/docs/release-control/v6/internal/subsystems/ai-runtime.md b/docs/release-control/v6/internal/subsystems/ai-runtime.md
index 16cc08de1b..0a75d2a053 100644
--- a/docs/release-control/v6/internal/subsystems/ai-runtime.md
+++ b/docs/release-control/v6/internal/subsystems/ai-runtime.md
@@ -8079,6 +8079,28 @@ inspection failure, stale facts, or mismatched identities remain
continues to project only `agent_attested` evidence, so adding the production
observer does not loosen the verification rule.
+### Incomplete investigation follow-up
+
+An empty or unusable final model response, provider error, or final-summary
+timeout does not constitute a model conclusion. The shared agentic loop retains
+a readable failure receipt and returns the original error identity for Patrol
+investigations. Interactive chat retains its existing best-effort fallback.
+The investigation service must not advance a captured proposal after that error.
+
+The private orchestrator clears the previous finding outcome when starting a
+new session, after retaining its prior context. Failed runs preserve evidence,
+counts and any existing action reference. They remain failed rather than being
+classified from fallback prose as a completed `needs_attention` investigation.
+Failed runs without a conclusion, including timeouts, may be assessed at the
+next scheduled Watch without another cooldown. Existing attempt limits,
+admission, autonomy and action authority still apply. A real completed model
+decision requiring human attention remains terminal.
+
+Proofs: `internal/ai/chat/agentic_final_failure_test.go`,
+`internal/ai/findings_provider_failure_test.go`, and the private
+`TestProviderFailureClearsPriorConclusionAndRetainsEvidence` regression.
+These causal tests do not themselves qualify model-owned unattended recovery.
+
### Proxmox guest lifecycle Patrol detector floor
`internal/ai/findings_proxmox_lifecycle.go` owns the deterministic production
diff --git a/internal/ai/chat/agentic.go b/internal/ai/chat/agentic.go
index bdfa617b14..961e9e2537 100644
--- a/internal/ai/chat/agentic.go
+++ b/internal/ai/chat/agentic.go
@@ -1464,8 +1464,7 @@ func (a *AgenticLoop) executeWithTools(ctx context.Context, sessionID string, me
}
log.Debug().Msg("agentic loop complete - no tool calls")
- resultMessages = a.ensureFinalTextResponse(ctx, sessionID, resultMessages, providerMessages, callback)
- return resultMessages, nil
+ return a.ensureFinalTextResponse(ctx, sessionID, resultMessages, providerMessages, callback)
}
// === Execute tool calls (three-phase pipeline) ===
@@ -1954,6 +1953,5 @@ func (a *AgenticLoop) executeWithTools(ctx context.Context, sessionID string, me
}
log.Warn().Int("max_turns", maxTurns).Str("session_id", sessionID).Msg("agentic loop hit max turns limit")
- resultMessages = a.ensureFinalTextResponse(ctx, sessionID, resultMessages, providerMessages, callback)
- return resultMessages, nil
+ return a.ensureFinalTextResponse(ctx, sessionID, resultMessages, providerMessages, callback)
}
diff --git a/internal/ai/chat/agentic_additional_test.go b/internal/ai/chat/agentic_additional_test.go
index 2ad8290c6c..ce8f48ddcd 100644
--- a/internal/ai/chat/agentic_additional_test.go
+++ b/internal/ai/chat/agentic_additional_test.go
@@ -1378,13 +1378,16 @@ func TestEnsureFinalTextResponse(t *testing.T) {
provider := &stubStreamingProvider{}
loop := &AgenticLoop{provider: provider, baseSystemPrompt: "prompt"}
- result := loop.ensureFinalTextResponse(
+ result, summaryErr := loop.ensureFinalTextResponse(
context.Background(),
"session-1",
[]Message{{Role: "assistant", Content: ""}},
[]providers.Message{{Role: "assistant", Content: ""}},
func(event StreamEvent) {},
)
+ if summaryErr != nil {
+ t.Fatal(summaryErr)
+ }
if len(result) != 2 {
t.Fatalf("expected summary message to be appended")
}
@@ -1398,7 +1401,7 @@ func TestEnsureFinalTextResponse(t *testing.T) {
provider.chatStream = func(ctx context.Context, req providers.ChatRequest, callback providers.StreamCallback) error {
return errors.New("boom")
}
- result2 := loop.ensureFinalTextResponse(
+ result2, summaryErr2 := loop.ensureFinalTextResponse(
context.Background(),
"session-2",
[]Message{
@@ -1408,6 +1411,9 @@ func TestEnsureFinalTextResponse(t *testing.T) {
[]providers.Message{{Role: "assistant", Content: ""}},
func(event StreamEvent) {},
)
+ if summaryErr2 != nil {
+ t.Fatal(summaryErr2)
+ }
if len(result2) != 3 {
t.Fatalf("expected fallback summary message when provider errors")
}
@@ -1422,7 +1428,7 @@ func TestEnsureFinalTextResponse(t *testing.T) {
callback(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{InputTokens: 1, OutputTokens: 1}})
return nil
}
- result3 := loop.ensureFinalTextResponse(
+ result3, summaryErr3 := loop.ensureFinalTextResponse(
context.Background(),
"session-3",
[]Message{
@@ -1432,6 +1438,9 @@ func TestEnsureFinalTextResponse(t *testing.T) {
[]providers.Message{{Role: "assistant", Content: ""}},
func(event StreamEvent) {},
)
+ if summaryErr3 != nil {
+ t.Fatal(summaryErr3)
+ }
if len(result3) != 3 {
t.Fatalf("expected fallback summary when provider returns empty content")
}
@@ -1453,7 +1462,7 @@ func TestEnsureFinalTextResponseRequiresAssistantTextAfterLatestToolResult(t *te
loop := &AgenticLoop{provider: provider, baseSystemPrompt: "prompt"}
var emitted strings.Builder
- result := loop.ensureFinalTextResponse(
+ result, summaryErr := loop.ensureFinalTextResponse(
context.Background(),
"session-pre-tool-preamble",
[]Message{
@@ -1494,6 +1503,9 @@ func TestEnsureFinalTextResponseRequiresAssistantTextAfterLatestToolResult(t *te
},
)
+ if summaryErr != nil {
+ t.Fatal(summaryErr)
+ }
if len(result) != 5 {
t.Fatalf("expected final summary to be appended after tool result, got %d messages", len(result))
}
diff --git a/internal/ai/chat/agentic_final.go b/internal/ai/chat/agentic_final.go
index 8e1119b753..15a66c7593 100644
--- a/internal/ai/chat/agentic_final.go
+++ b/internal/ai/chat/agentic_final.go
@@ -29,7 +29,7 @@ func (a *AgenticLoop) ensureFinalTextResponse(
resultMessages []Message,
providerMessages []providers.Message,
callback StreamCallback,
-) []Message {
+) ([]Message, error) {
return a.ensureFinalTextResponseWithSystemPrompt(ctx, sessionID, resultMessages, providerMessages, callback, "")
}
@@ -43,9 +43,9 @@ func (a *AgenticLoop) ensureFinalTextResponseWithSystemPrompt(
providerMessages []providers.Message,
callback StreamCallback,
systemPromptOverride string,
-) []Message {
+) ([]Message, error) {
if hasFinalAssistantText(resultMessages) {
- return resultMessages
+ return resultMessages, nil
}
// No text content from the model. Make a final text-only call.
@@ -136,16 +136,17 @@ func (a *AgenticLoop) ensureFinalTextResponseWithSystemPrompt(
}
}
- if summaryErr == nil && summaryBuilder.Len() > 0 {
+ summaryText := cleanToolCallArtifacts(summaryBuilder.String())
+ if summaryErr == nil && !suppressLeakedToolContent && strings.TrimSpace(summaryText) != "" {
summaryMsg := Message{
ID: uuid.New().String(),
Role: "assistant",
- Content: cleanToolCallArtifacts(summaryBuilder.String()),
+ Content: summaryText,
Timestamp: time.Now(),
}
resultMessages = append(resultMessages, summaryMsg)
log.Info().Str("session_id", sessionID).Int("summary_len", summaryBuilder.Len()).Msg("[AgenticLoop] Final summary produced")
- return resultMessages
+ return resultMessages, nil
}
if summaryErr != nil {
@@ -157,6 +158,9 @@ func (a *AgenticLoop) ensureFinalTextResponseWithSystemPrompt(
// Deterministic fallback so the user always gets a usable response even if
// the model fails to emit final text.
fallback := buildAutomaticFallbackSummary(resultMessages)
+ if isPatrolInvestigationExecution(a.currentExecutionProfile()) {
+ fallback = "The model did not return a final narrative for this investigation. Evidence remains in the investigation history. Patrol can reassess this finding on a later scheduled check within the current policy and attempt limit."
+ }
fallbackMsg := Message{
ID: uuid.New().String(),
Role: "assistant",
@@ -168,7 +172,18 @@ func (a *AgenticLoop) ensureFinalTextResponseWithSystemPrompt(
callback(StreamEvent{Type: "content", Data: jsonData})
log.Warn().Str("session_id", sessionID).Int("summary_len", len(fallback)).Msg("[AgenticLoop] Emitted deterministic fallback summary")
- return resultMessages
+ // A readable receipt is not a completed model investigation. Preserve it
+ // for the operator, but keep failure on the error channel so the caller
+ // cannot classify fallback prose as a request for human intervention or
+ // advance a captured action proposal. Interactive chat keeps its existing
+ // best-effort narrative behaviour.
+ if isPatrolInvestigationExecution(a.currentExecutionProfile()) {
+ if summaryErr != nil {
+ return resultMessages, fmt.Errorf("investigation final response failed: %w", summaryErr)
+ }
+ return resultMessages, fmt.Errorf("investigation final response was empty or unusable")
+ }
+ return resultMessages, nil
}
func hasFinalAssistantText(resultMessages []Message) bool {
diff --git a/internal/ai/chat/agentic_final_failure_test.go b/internal/ai/chat/agentic_final_failure_test.go
new file mode 100644
index 0000000000..60d981715d
--- /dev/null
+++ b/internal/ai/chat/agentic_final_failure_test.go
@@ -0,0 +1,104 @@
+package chat
+
+import (
+ "context"
+ "errors"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/ai/providers"
+ "github.com/rcourtman/pulse-go-rewrite/internal/ai/tools"
+)
+
+func TestInvestigationFinalResponseFailureRetainsReceiptAndError(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ err error
+ }{
+ {"empty", nil}, {"timeout", context.DeadlineExceeded}, {"provider", errors.New("provider unavailable")},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ calls := 0
+ provider := &stubStreamingProvider{chatStream: func(ctx context.Context, req providers.ChatRequest, callback providers.StreamCallback) error {
+ calls++
+ if calls == 1 {
+ callback(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{StopReason: "tool_use"}})
+ return nil
+ }
+ if len(req.Tools) != 0 {
+ t.Fatal("summary offered write tools")
+ }
+ return tc.err
+ }}
+ loop := NewAgenticLoop(provider, tools.NewPulseToolExecutor(tools.ExecutorConfig{}), "Investigate current evidence")
+ loop.SetExecutionProfile(tools.ProfilePatrolInvestigation)
+ messages, err := loop.ExecuteWithTools(context.Background(), "incomplete", []Message{{Role: "user", Content: "Check service"}}, nil, func(StreamEvent) {})
+ if err == nil {
+ t.Fatal("incomplete provider run was reported as successful")
+ }
+ if tc.err != nil && !errors.Is(err, tc.err) {
+ t.Fatalf("lost error identity: %v", err)
+ }
+ if calls != 2 {
+ t.Fatalf("provider calls=%d, want original plus bounded summary", calls)
+ }
+ if len(messages) == 0 || !strings.Contains(messages[len(messages)-1].Content, "final narrative") {
+ t.Fatalf("missing retained fallback receipt: %#v", messages)
+ }
+ })
+ }
+}
+
+func TestInvestigationEmptyResponseCanRecoverWithRealModelConclusion(t *testing.T) {
+ calls := 0
+ p := &stubStreamingProvider{chatStream: func(ctx context.Context, req providers.ChatRequest, callback providers.StreamCallback) error {
+ calls++
+ if calls == 2 {
+ callback(providers.StreamEvent{Type: "content", Data: providers.ContentEvent{Text: "### Conclusion\nNEEDS_ATTENTION: upstream owner must repair the dependency"}})
+ }
+ callback(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{}})
+ return nil
+ }}
+ loop := NewAgenticLoop(p, tools.NewPulseToolExecutor(tools.ExecutorConfig{}), "Investigate")
+ loop.SetExecutionProfile(tools.ProfilePatrolInvestigation)
+ messages, err := loop.ExecuteWithTools(context.Background(), "conclusion", []Message{{Role: "user", Content: "Check"}}, nil, func(StreamEvent) {})
+ if err != nil || !strings.Contains(messages[len(messages)-1].Content, "NEEDS_ATTENTION:") {
+ t.Fatalf("real model conclusion was discarded: %v %#v", err, messages)
+ }
+}
+
+func TestInvestigationSummaryRetainsCallerDeadline(t *testing.T) {
+ calls := 0
+ p := &stubStreamingProvider{chatStream: func(ctx context.Context, req providers.ChatRequest, cb providers.StreamCallback) error {
+ calls++
+ if calls == 1 {
+ cb(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{}})
+ return nil
+ }
+ <-ctx.Done()
+ return ctx.Err()
+ }}
+ loop := NewAgenticLoop(p, tools.NewPulseToolExecutor(tools.ExecutorConfig{}), "Investigate")
+ loop.SetExecutionProfile(tools.ProfilePatrolInvestigation)
+ ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
+ defer cancel()
+ _, err := loop.ExecuteWithTools(ctx, "deadline", []Message{{Role: "user", Content: "Check"}}, nil, func(StreamEvent) {})
+ if !errors.Is(err, context.DeadlineExceeded) {
+ t.Fatalf("deadline identity lost: %v", err)
+ }
+}
+
+func TestInvestigationWhitespaceSummaryIsNotAConclusion(t *testing.T) {
+ p := &stubStreamingProvider{chatStream: func(ctx context.Context, req providers.ChatRequest, cb providers.StreamCallback) error {
+ cb(providers.StreamEvent{Type: "content", Data: providers.ContentEvent{Text: " \n "}})
+ cb(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{}})
+ return nil
+ }}
+ loop := NewAgenticLoop(p, tools.NewPulseToolExecutor(tools.ExecutorConfig{}), "Investigate")
+ loop.SetExecutionProfile(tools.ProfilePatrolInvestigation)
+ _, err := loop.ExecuteWithTools(context.Background(), "whitespace", []Message{{Role: "user", Content: "Check"}}, nil, func(StreamEvent) {})
+ if err == nil {
+ t.Fatal("whitespace summary became a completed investigation")
+ }
+}
diff --git a/internal/ai/findings.go b/internal/ai/findings.go
index 15d32e60ab..1c0fefb1cb 100644
--- a/internal/ai/findings.go
+++ b/internal/ai/findings.go
@@ -464,6 +464,17 @@ func investigationCooldownForOutcome(outcome string) time.Duration {
}
}
+func (f *Finding) investigationCooldown() time.Duration {
+ if InvestigationStatus(f.InvestigationStatus) == InvestigationStatusFailed &&
+ (f.InvestigationOutcome == "" || InvestigationOutcome(f.InvestigationOutcome) == InvestigationOutcomeTimedOut) {
+ // The run failed before producing a conclusion. The scheduled Watch
+ // cadence owns the next assessment; there is no additional cooldown.
+ // Admission, attempt limits and action authority still apply.
+ return 0
+ }
+ return investigationCooldownForOutcome(f.InvestigationOutcome)
+}
+
func deriveLoopState(f *Finding) FindingLoopState {
if f == nil {
return FindingLoopStateDetected
@@ -575,7 +586,7 @@ func (f *Finding) ShouldInvestigate(autonomyLevel string) bool {
// Don't re-investigate within cooldown period
// Timeout failures use a shorter cooldown (10min) since they're transient
if f.LastInvestigatedAt != nil {
- cooldown := investigationCooldownForOutcome(f.InvestigationOutcome)
+ cooldown := f.investigationCooldown()
if time.Since(*f.LastInvestigatedAt) < cooldown {
return false
}
@@ -677,7 +688,7 @@ func (f *Finding) CanRetryInvestigation() bool {
// Can't retry if still in cooldown
// Timeout failures use a shorter cooldown (10min) since they're transient
if f.LastInvestigatedAt != nil {
- cooldown := investigationCooldownForOutcome(f.InvestigationOutcome)
+ cooldown := f.investigationCooldown()
if time.Since(*f.LastInvestigatedAt) < cooldown {
return false
}
diff --git a/internal/ai/findings_provider_failure_test.go b/internal/ai/findings_provider_failure_test.go
new file mode 100644
index 0000000000..3530ad4e62
--- /dev/null
+++ b/internal/ai/findings_provider_failure_test.go
@@ -0,0 +1,46 @@
+package ai
+
+import (
+ "github.com/rcourtman/pulse-go-rewrite/internal/config"
+ "testing"
+ "time"
+)
+
+func TestProviderFailureFollowupSurvivesRestart(t *testing.T) {
+ for _, outcome := range []string{"", string(InvestigationOutcomeTimedOut)} {
+ t.Run("outcome_"+outcome, func(t *testing.T) {
+ persistence := config.NewConfigPersistence(t.TempDir())
+ store := NewFindingsStore()
+ if err := store.SetPersistence(NewFindingsPersistenceAdapter(persistence)); err != nil {
+ t.Fatal(err)
+ }
+ now := time.Now()
+ store.Add(&Finding{ID: "failed-provider", ResourceID: "app-container:service", ResourceType: "app-container", Title: "Service unavailable", Severity: FindingSeverityWarning, Category: FindingCategoryReliability, InvestigationSessionID: "failed-run", InvestigationStatus: string(InvestigationStatusFailed), InvestigationOutcome: outcome, LastInvestigatedAt: &now, InvestigationAttempts: 1})
+ if err := store.ForceSave(); err != nil {
+ t.Fatal(err)
+ }
+ restored := NewFindingsStore()
+ if err := restored.SetPersistence(NewFindingsPersistenceAdapter(persistence)); err != nil {
+ t.Fatal(err)
+ }
+ f := restored.Get("failed-provider")
+ if !f.ShouldInvestigate("full") || !f.CanRetryInvestigation() {
+ t.Fatal("persisted provider failure delayed the next scheduled model assessment")
+ }
+ f.InvestigationAttempts = maxInvestigationAttempts
+ if f.ShouldInvestigate("full") {
+ t.Fatal("bypassed attempt limit")
+ }
+ f.InvestigationAttempts = 1
+ f.InvestigationStatus = string(InvestigationStatusRunning)
+ if f.ShouldInvestigate("full") {
+ t.Fatal("duplicated running investigation")
+ }
+ f.InvestigationStatus = string(InvestigationStatusCompleted)
+ f.InvestigationOutcome = string(InvestigationOutcomeNeedsAttention)
+ if f.ShouldInvestigate("full") {
+ t.Fatal("overrode real human-attention conclusion")
+ }
+ })
+ }
+}
diff --git a/internal/ai/findings_timeout_test.go b/internal/ai/findings_timeout_test.go
index 6c9f3aff28..e0d70c2165 100644
--- a/internal/ai/findings_timeout_test.go
+++ b/internal/ai/findings_timeout_test.go
@@ -5,9 +5,8 @@ import (
"time"
)
-func TestShouldInvestigate_TimeoutUsesShortCooldown(t *testing.T) {
- // Finding with timed_out outcome and LastInvestigatedAt 15 minutes ago
- // should be eligible for re-investigation (shorter 10min cooldown)
+func TestShouldInvestigate_FailedTimeoutUsesScheduledCadence(t *testing.T) {
+ // Failed timeouts use the scheduled assessment cadence.
fifteenMinsAgo := time.Now().Add(-15 * time.Minute)
f := &Finding{
ID: "f1",
@@ -24,18 +23,18 @@ func TestShouldInvestigate_TimeoutUsesShortCooldown(t *testing.T) {
t.Fatalf("expected timed-out finding (15min old) to be eligible for re-investigation")
}
- // Same finding but only 5 minutes ago — still within the 10min timeout cooldown
+ // A failed run follows the Watch cadence without an additional cooldown.
fiveMinsAgo := time.Now().Add(-5 * time.Minute)
f.LastInvestigatedAt = &fiveMinsAgo
- if f.ShouldInvestigate("approval") {
- t.Fatalf("expected timed-out finding (5min old) to still be in cooldown")
+ if !f.ShouldInvestigate("approval") {
+ t.Fatalf("failed timeout should be eligible at the next scheduled assessment")
}
}
-func TestShouldInvestigate_NonTimeoutKeepsLongCooldown(t *testing.T) {
+func TestShouldInvestigate_FailedProviderUsesScheduledCadence(t *testing.T) {
// Finding with no special outcome (regular failure) at 15 minutes ago
- // should NOT be eligible — still within the 1-hour cooldown
+ // remains eligible for the next scheduled assessment.
fifteenMinsAgo := time.Now().Add(-15 * time.Minute)
f := &Finding{
ID: "f1",
@@ -48,8 +47,8 @@ func TestShouldInvestigate_NonTimeoutKeepsLongCooldown(t *testing.T) {
InvestigationAttempts: 1,
}
- if f.ShouldInvestigate("approval") {
- t.Fatalf("expected non-timeout failure (15min old) to still be in 1hr cooldown")
+ if !f.ShouldInvestigate("approval") {
+ t.Fatalf("failed provider run should not wait another hour")
}
// After 2 hours, should be eligible