Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

fix: keep streamed chat summary prefix before a leaked tool call

rcourtman3434e44d523a
diff --git a/docs/release-control/v6/internal/subsystems/ai-runtime.md b/docs/release-control/v6/internal/subsystems/ai-runtime.md
index 4bad35d0e9..1ced8c9a36 100644
--- a/docs/release-control/v6/internal/subsystems/ai-runtime.md
+++ b/docs/release-control/v6/internal/subsystems/ai-runtime.md
@@ -8087,6 +8087,15 @@ a readable failure receipt and returns the original error identity for Patrol
investigations. Interactive chat retains its existing best-effort fallback.
The investigation service must not advance a captured proposal after that error.
+A final summary that leaks a textual tool call is unusable only for Patrol
+investigations: the model was still investigating, and the orchestrator would
+read the unmarked prefix as a terminal `needs_attention` conclusion. Interactive
+chat keeps the non-empty clean prefix it already streamed as the final message,
+so the persisted message matches the stream, and falls back only when that
+prefix is empty. Proof: `TestInteractiveFinalSummaryKeepsCleanPrefixBeforeLeakedToolCall`
+and `TestInvestigationFinalSummaryWithLeakedToolCallIsNotAConclusion` in
+`internal/ai/chat/agentic_final_failure_test.go`.
+
The private orchestrator clears the previous finding outcome when starting a
new session, after retaining its prior context. Failed runs preserve evidence,
counts and any existing action reference. They remain failed rather than being
diff --git a/internal/ai/chat/agentic_final.go b/internal/ai/chat/agentic_final.go
index 15a66c7593..c162f673b5 100644
--- a/internal/ai/chat/agentic_final.go
+++ b/internal/ai/chat/agentic_final.go
@@ -137,7 +137,14 @@ func (a *AgenticLoop) ensureFinalTextResponseWithSystemPrompt(
}
summaryText := cleanToolCallArtifacts(summaryBuilder.String())
- if summaryErr == nil && !suppressLeakedToolContent && strings.TrimSpace(summaryText) != "" {
+ // A textual tool call in this tool-free turn means the model was still
+ // investigating rather than concluding. Interactive chat keeps the clean
+ // prefix it has already streamed, so the persisted message matches what
+ // the user saw. A Patrol investigation must not: the orchestrator reads
+ // unmarked prose as NEEDS_ATTENTION, a terminal conclusion, so it takes
+ // the failure receipt and error below and stays eligible for follow-up.
+ rejectLeakedSummary := suppressLeakedToolContent && isPatrolInvestigationExecution(a.currentExecutionProfile())
+ if summaryErr == nil && !rejectLeakedSummary && strings.TrimSpace(summaryText) != "" {
summaryMsg := Message{
ID: uuid.New().String(),
Role: "assistant",
@@ -151,6 +158,8 @@ func (a *AgenticLoop) ensureFinalTextResponseWithSystemPrompt(
if summaryErr != nil {
log.Error().Err(summaryErr).Str("session_id", sessionID).Msg("[AgenticLoop] Final summary call failed")
+ } else if rejectLeakedSummary {
+ log.Warn().Str("session_id", sessionID).Msg("[AgenticLoop] Final investigation summary leaked a tool call instead of concluding")
} else {
log.Warn().Str("session_id", sessionID).Msg("[AgenticLoop] Final summary call returned empty content")
}
diff --git a/internal/ai/chat/agentic_final_failure_test.go b/internal/ai/chat/agentic_final_failure_test.go
index 60d981715d..be5d6e427d 100644
--- a/internal/ai/chat/agentic_final_failure_test.go
+++ b/internal/ai/chat/agentic_final_failure_test.go
@@ -2,6 +2,7 @@ package chat
import (
"context"
+ "encoding/json"
"errors"
"strings"
"testing"
@@ -102,3 +103,73 @@ func TestInvestigationWhitespaceSummaryIsNotAConclusion(t *testing.T) {
t.Fatal("whitespace summary became a completed investigation")
}
}
+
+// leakedToolCallSummaryProvider makes the first turn end without text, then
+// streams a clean narrative prefix followed by a leaked textual tool call in
+// the tool-free final summary turn.
+func leakedToolCallSummaryProvider(prefix string) *stubStreamingProvider {
+ calls := 0
+ return &stubStreamingProvider{chatStream: func(ctx context.Context, req providers.ChatRequest, cb providers.StreamCallback) error {
+ calls++
+ if calls == 2 {
+ cb(providers.StreamEvent{Type: "content", Data: providers.ContentEvent{Text: prefix}})
+ cb(providers.StreamEvent{Type: "content", Data: providers.ContentEvent{Text: `pulse_read(target_host="current_resource", command="df -h")`}})
+ }
+ cb(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{}})
+ return nil
+ }}
+}
+
+func streamedContent(t *testing.T, events []StreamEvent) string {
+ t.Helper()
+ var streamed strings.Builder
+ for _, event := range events {
+ if event.Type != "content" {
+ continue
+ }
+ var data ContentData
+ if err := json.Unmarshal(event.Data, &data); err != nil {
+ t.Fatalf("decode content event: %v", err)
+ }
+ streamed.WriteString(data.Text)
+ }
+ return streamed.String()
+}
+
+func TestInteractiveFinalSummaryKeepsCleanPrefixBeforeLeakedToolCall(t *testing.T) {
+ prefix := "The root filesystem on pve1 is 92% full, driven by /var/log. "
+ loop := NewAgenticLoop(leakedToolCallSummaryProvider(prefix), tools.NewPulseToolExecutor(tools.ExecutorConfig{}), "Answer the user")
+ var events []StreamEvent
+ messages, err := loop.ExecuteWithTools(context.Background(), "leaked-chat", []Message{{Role: "user", Content: "Why is pve1 alerting?"}}, nil, func(event StreamEvent) {
+ events = append(events, event)
+ })
+ if err != nil {
+ t.Fatalf("interactive chat failed on a usable narrative: %v", err)
+ }
+ if len(messages) == 0 {
+ t.Fatal("no messages returned")
+ }
+ final := messages[len(messages)-1].Content
+ if strings.TrimSpace(final) != strings.TrimSpace(prefix) {
+ t.Fatalf("persisted final message = %q, want the clean streamed prefix %q", final, prefix)
+ }
+ if streamed := streamedContent(t, events); streamed != final {
+ t.Fatalf("streamed content %q differs from persisted message %q", streamed, final)
+ }
+}
+
+func TestInvestigationFinalSummaryWithLeakedToolCallIsNotAConclusion(t *testing.T) {
+ // A textual tool call in the tool-free summary turn means the model was
+ // still investigating. The orchestrator classifies unmarked prose as
+ // NEEDS_ATTENTION, which is terminal, so the prefix must not become the
+ // investigation's conclusion.
+ loop := NewAgenticLoop(leakedToolCallSummaryProvider("I will check disk usage on pve1 next. "), tools.NewPulseToolExecutor(tools.ExecutorConfig{}), "Investigate")
+ loop.SetExecutionProfile(tools.ProfilePatrolInvestigation)
+ messages, err := loop.ExecuteWithTools(context.Background(), "leaked-investigation", []Message{{Role: "user", Content: "Check"}}, nil, func(StreamEvent) {})
+ if err == nil {
+ t.Fatal("leaked tool-call summary became a completed investigation")
+ }
+ if len(messages) == 0 || !strings.Contains(messages[len(messages)-1].Content, "final narrative") {
+ t.Fatalf("missing retained fallback receipt: %#v", messages)
+ }
+}