Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package chat
import (
"context"
"encoding/json"
"fmt"
"sort"
"strings"
"time"
"github.com/google/uuid"
"github.com/rcourtman/pulse-go-rewrite/internal/ai/providers"
"github.com/rs/zerolog/log"
)
// ensureFinalTextResponse checks if the result messages contain assistant text
// after the latest user/tool-result anchor. If not, it makes one last LLM call
// without tools so the model can summarize its findings.
// This prevents the loop from exiting silently after making tool calls without answering.
//
// cost-recording-exempt: any tokens this final summary turn consumes
// flow into a.totalInputTokens / a.totalOutputTokens via the done
// event, and the orchestrator (chat.Service.recordChatTurnCost)
// records the loop totals after ExecuteWithTools returns. Recording
// here would double-count.
func (a *AgenticLoop) ensureFinalTextResponse(
ctx context.Context,
sessionID string,
resultMessages []Message,
providerMessages []providers.Message,
callback StreamCallback,
) ([]Message, error) {
return a.ensureFinalTextResponseWithSystemPrompt(ctx, sessionID, resultMessages, providerMessages, callback, "")
}
// cost-recording-exempt: any tokens this final summary turn consumes flow into
// a.totalInputTokens / a.totalOutputTokens via the done event, and the
// orchestrator records the loop totals after ExecuteWithTools returns.
func (a *AgenticLoop) ensureFinalTextResponseWithSystemPrompt(
ctx context.Context,
sessionID string,
resultMessages []Message,
providerMessages []providers.Message,
callback StreamCallback,
systemPromptOverride string,
) ([]Message, error) {
if hasFinalAssistantText(resultMessages) {
return resultMessages, nil
}
// No text content from the model. Make a final text-only call.
log.Warn().Str("session_id", sessionID).Msg("[AgenticLoop] No text content produced — making final summary call")
// Build clean message history for the summary call:
// 1. Strip any trailing empty assistant messages (the model already failed to produce
// text with these, so including them would just get the same empty result).
// 2. Append a user-role nudge to give the model a clear instruction.
cleanMessages := make([]providers.Message, len(providerMessages))
copy(cleanMessages, providerMessages)
for len(cleanMessages) > 0 {
last := cleanMessages[len(cleanMessages)-1]
if last.Role == "assistant" && strings.TrimSpace(last.Content) == "" && len(last.ToolCalls) == 0 {
cleanMessages = cleanMessages[:len(cleanMessages)-1]
} else {
break
}
}
cleanMessages = append(cleanMessages, providers.Message{
Role: "user",
Content: "Based on what you've investigated above, provide a complete response to the user. Explain what you found or did, mention any issues or caveats they should know about, and suggest next steps if relevant.",
})
summarySystemPrompt := a.getSystemPrompt()
if override := strings.TrimSpace(systemPromptOverride); override != "" {
summarySystemPrompt = override
}
summaryReq := providers.ChatRequest{
Messages: cleanMessages,
System: summarySystemPrompt,
ExecutionID: a.executionID,
// No Tools field: this is a final narrative turn, and Pulse avoids
// provider-specific tool_choice transport fields.
}
applyExecutionInferenceAllowance(&summaryReq, a.currentExecutionProfile(), true, a.totalOutputTokens)
applyExecutionRuntimeEnvelope(&summaryReq, a.currentExecutionProfile())
a.mu.Lock()
requestSanitizer := a.requestSanitizer
a.mu.Unlock()
if requestSanitizer != nil {
summaryReq = requestSanitizer(summaryReq)
}
var summaryBuilder strings.Builder
var suppressLeakedToolContent bool
var pendingVisibleContent string
// Keep this bounded so a stuck provider stream doesn't drag the whole
// request. An investigation may use the time left before its deadline.
summaryCtx, cancel := context.WithTimeout(ctx, investigationSummaryTimeout(ctx, a.currentExecutionProfile(), 20*time.Second))
defer cancel()
summaryErr := a.provider.ChatStream(summaryCtx, summaryReq, func(event providers.StreamEvent) {
switch event.Type {
case "content":
if data, ok := event.Data.(providers.ContentEvent); ok {
if suppressLeakedToolContent {
return
}
visibleText, leakFound := appendVisibleContentBeforeToolLeak(&summaryBuilder, &pendingVisibleContent, data.Text)
if visibleText != "" {
jsonData, _ := json.Marshal(ContentData{Text: visibleText})
callback(StreamEvent{Type: "content", Data: jsonData})
}
if leakFound {
suppressLeakedToolContent = true
}
}
case "done":
if data, ok := event.Data.(providers.DoneEvent); ok {
a.totalInputTokens += data.InputTokens
a.totalCacheCreationTokens += data.CacheCreationInputTokens
a.totalCacheReadTokens += data.CacheReadInputTokens
a.totalOutputTokens += data.OutputTokens
}
}
})
if summaryErr == nil {
a.totalModelTurns++
}
if summaryErr == nil && !suppressLeakedToolContent {
if visibleText := flushPendingVisibleContent(&summaryBuilder, &pendingVisibleContent); visibleText != "" {
jsonData, _ := json.Marshal(ContentData{Text: visibleText})
callback(StreamEvent{Type: "content", Data: jsonData})
}
}
summaryText := cleanToolCallArtifacts(summaryBuilder.String())
// A textual tool call in this tool-free turn means the model was still
// investigating rather than concluding. Interactive chat keeps the clean
// prefix it has already streamed, so the persisted message matches what
// the user saw. A Patrol investigation must not: the orchestrator reads
// unmarked prose as NEEDS_ATTENTION, a terminal conclusion, so it takes
// the failure receipt and error below and stays eligible for follow-up.
rejectLeakedSummary := suppressLeakedToolContent && isPatrolInvestigationExecution(a.currentExecutionProfile())
if summaryErr == nil && !rejectLeakedSummary && strings.TrimSpace(summaryText) != "" {
summaryMsg := Message{
ID: uuid.New().String(),
Role: "assistant",
Content: summaryText,
Timestamp: time.Now(),
}
resultMessages = append(resultMessages, summaryMsg)
log.Info().Str("session_id", sessionID).Int("summary_len", summaryBuilder.Len()).Msg("[AgenticLoop] Final summary produced")
return resultMessages, nil
}
if summaryErr != nil {
log.Error().Err(summaryErr).Str("session_id", sessionID).Msg("[AgenticLoop] Final summary call failed")
} else if rejectLeakedSummary {
log.Warn().Str("session_id", sessionID).Msg("[AgenticLoop] Final investigation summary leaked a tool call instead of concluding")
} else {
log.Warn().Str("session_id", sessionID).Msg("[AgenticLoop] Final summary call returned empty content")
}
// Deterministic fallback so the user always gets a usable response even if
// the model fails to emit final text.
fallback := buildAutomaticFallbackSummary(resultMessages)
if isPatrolInvestigationExecution(a.currentExecutionProfile()) {
fallback = "The model did not return a final narrative for this investigation. Evidence remains in the investigation history. Patrol can reassess this finding on a later scheduled check within the current policy and attempt limit."
}
fallbackMsg := Message{
ID: uuid.New().String(),
Role: "assistant",
Content: fallback,
Timestamp: time.Now(),
}
resultMessages = append(resultMessages, fallbackMsg)
jsonData, _ := json.Marshal(ContentData{Text: fallback})
callback(StreamEvent{Type: "content", Data: jsonData})
log.Warn().Str("session_id", sessionID).Int("summary_len", len(fallback)).Msg("[AgenticLoop] Emitted deterministic fallback summary")
// A readable receipt is not a completed model investigation. Preserve it
// for the operator, but keep failure on the error channel so the caller
// cannot classify fallback prose as a request for human intervention or
// advance a captured action proposal. Interactive chat keeps its existing
// best-effort narrative behaviour.
if isPatrolInvestigationExecution(a.currentExecutionProfile()) {
if summaryErr != nil {
return resultMessages, fmt.Errorf("investigation final response failed: %w", summaryErr)
}
return resultMessages, fmt.Errorf("investigation final response was empty or unusable")
}
return resultMessages, nil
}
func hasFinalAssistantText(resultMessages []Message) bool {
anchor := -1
for i, msg := range resultMessages {
if msg.ToolResult != nil {
anchor = i
continue
}
if msg.Role == "user" && strings.TrimSpace(msg.Content) != "" {
anchor = i
}
}
for i := len(resultMessages) - 1; i > anchor; i-- {
if resultMessages[i].Role == "assistant" && strings.TrimSpace(resultMessages[i].Content) != "" {
return true
}
}
return false
}
func buildAutomaticFallbackSummary(resultMessages []Message) string {
successCount := 0
errorCount := 0
toolCounts := make(map[string]int)
// Resolve provider tool-call ids to the real tool name so the summary names
// the checks (query, metrics, ...) instead of leaking raw `call_…` ids.
idToName := make(map[string]string)
idToCall := make(map[string]ToolCall)
for _, msg := range resultMessages {
for _, call := range msg.ToolCalls {
id := strings.TrimSpace(call.ID)
name := strings.TrimSpace(call.Name)
if id != "" && name != "" {
idToName[id] = name
idToCall[id] = call
}
}
}
var proposal *automaticFallbackProposal
for _, msg := range resultMessages {
if msg.ToolResult == nil {
continue
}
if msg.ToolResult.IsError {
errorCount++
continue
}
successCount++
name := fallbackToolDisplayName(msg.ToolResult.ToolUseID, idToName)
if name != "" {
toolCounts[name]++
}
if strings.EqualFold(name, "patrol_propose_action") {
if call, ok := idToCall[strings.TrimSpace(msg.ToolResult.ToolUseID)]; ok {
proposal = automaticFallbackProposalFromToolCall(call)
}
}
}
if successCount == 0 && errorCount == 0 {
return "I couldn't generate a final narrative response from the model in this run. Please retry, and I can run the checks again."
}
type toolCount struct {
Name string
Count int
}
var ranked []toolCount
for name, count := range toolCounts {
ranked = append(ranked, toolCount{Name: name, Count: count})
}
sort.Slice(ranked, func(i, j int) bool {
if ranked[i].Count == ranked[j].Count {
return ranked[i].Name < ranked[j].Name
}
return ranked[i].Count > ranked[j].Count
})
toolList := "infrastructure tools"
if len(ranked) > 0 {
names := make([]string, 0, len(ranked))
maxNames := 4
if len(ranked) < maxNames {
maxNames = len(ranked)
}
for i := 0; i < maxNames; i++ {
names = append(names, ranked[i].Name)
}
toolList = strings.Join(names, ", ")
}
if finding, ok := automaticFallbackFindingFromMessages(resultMessages); ok {
return buildInvestigationFallbackSummary(finding, proposal, successCount, errorCount, toolList)
}
checkWord := "checks"
if successCount == 1 {
checkWord = "check"
}
summary := fmt.Sprintf(
"I ran %d %s (%s) but the model didn't return a written summary this time. Ask me again and I'll pull the results together.",
successCount, checkWord, toolList,
)
if errorCount > 0 {
errWord := "errors"
if errorCount == 1 {
errWord = "error"
}
summary += fmt.Sprintf(" %d tool %s occurred during the run.", errorCount, errWord)
}
return summary
}
type automaticFallbackFinding struct {
Title string
Severity string
Category string
ResourceName string
ResourceID string
ResourceType string
Description string
}
type automaticFallbackProposal struct {
CapabilityName string
ResourceID string
}
func automaticFallbackFindingFromMessages(resultMessages []Message) (automaticFallbackFinding, bool) {
for i := len(resultMessages) - 1; i >= 0; i-- {
msg := resultMessages[i]
if msg.Role != "user" || msg.ToolResult != nil {
continue
}
if finding, ok := parseAutomaticFallbackFinding(msg.Content); ok {
return finding, true
}
}
return automaticFallbackFinding{}, false
}
func parseAutomaticFallbackFinding(prompt string) (automaticFallbackFinding, bool) {
const sectionHeader = "## Finding Details"
sectionStart := strings.Index(prompt, sectionHeader)
if sectionStart < 0 {
return automaticFallbackFinding{}, false
}
finding := automaticFallbackFinding{}
for _, rawLine := range strings.Split(prompt[sectionStart+len(sectionHeader):], "\n") {
line := strings.TrimSpace(rawLine)
if strings.HasPrefix(line, "## ") {
break
}
value := func(label string) (string, bool) {
prefix := "- **" + label + "**:"
if !strings.HasPrefix(line, prefix) {
return "", false
}
return automaticFallbackFact(strings.TrimSpace(strings.TrimPrefix(line, prefix))), true
}
if v, ok := value("Title"); ok && finding.Title == "" {
finding.Title = v
} else if v, ok := value("Severity"); ok && finding.Severity == "" {
finding.Severity = v
} else if v, ok := value("Category"); ok && finding.Category == "" {
finding.Category = v
} else if v, ok := value("Resource display name"); ok && finding.ResourceName == "" {
finding.ResourceName = v
} else if v, ok := value("Canonical resource ID"); ok && finding.ResourceID == "" {
finding.ResourceID = v
} else if v, ok := value("Resource type"); ok && finding.ResourceType == "" {
finding.ResourceType = v
} else if v, ok := value("Description"); ok && finding.Description == "" {
finding.Description = v
}
}
if finding.Title == "" || finding.ResourceID == "" {
return automaticFallbackFinding{}, false
}
return finding, true
}
func automaticFallbackFact(value string) string {
value = cleanToolCallArtifacts(value)
value = strings.Join(strings.Fields(value), " ")
const maxRunes = 320
runes := []rune(value)
if len(runes) > maxRunes {
value = string(runes[:maxRunes-1]) + "…"
}
return value
}
func automaticFallbackProposalFromToolCall(call ToolCall) *automaticFallbackProposal {
capabilityName, _ := call.Input["capability_name"].(string)
resourceID, _ := call.Input["resource_id"].(string)
capabilityName = automaticFallbackFact(capabilityName)
resourceID = automaticFallbackFact(resourceID)
if capabilityName == "" || resourceID == "" {
return nil
}
return &automaticFallbackProposal{CapabilityName: capabilityName, ResourceID: resourceID}
}
func buildInvestigationFallbackSummary(finding automaticFallbackFinding, proposal *automaticFallbackProposal, successCount, errorCount int, toolList string) string {
resourceName := finding.ResourceName
if resourceName == "" {
resourceName = finding.ResourceID
}
resourceDetails := finding.ResourceID
if finding.ResourceType != "" {
resourceDetails += "; " + finding.ResourceType
}
checkWord := "tool calls"
if successCount == 1 {
checkWord = "tool call"
}
errorNote := ""
if errorCount > 0 {
errorWord := "errors"
if errorCount == 1 {
errorWord = "error"
}
errorNote = fmt.Sprintf(" %d tool %s were also recorded; they are not being treated as evidence.", errorCount, errorWord)
}
symptom := finding.Description
if symptom == "" {
symptom = finding.Title
}
classification := ""
if finding.Severity != "" || finding.Category != "" {
classification = strings.Trim(strings.Join([]string{finding.Severity, finding.Category}, " / "), " / ")
if classification != "" {
classification = " Classification: " + classification + "."
}
}
recommendation := "Review the recorded evidence before choosing a manual next step."
proposalConclusion := "No governed action proposal was recorded, so nothing was authorized by this fallback."
if proposal != nil {
recommendation = fmt.Sprintf("A governed %s proposal was recorded for %s. Core policy—not this fallback—decides whether it needs approval or may run automatically.", proposal.CapabilityName, proposal.ResourceID)
proposalConclusion = "The proposal remains traceable through its governed action receipt; this fallback does not infer execution or resolution."
}
return fmt.Sprintf(`### Investigation Summary
Patrol investigated %s on %s. It completed %d successful %s using %s.%s The model did not return a final narrative, so this deterministic receipt uses only the recorded finding and successful tool outcomes.
### Root Cause
Patrol cannot claim a confirmed root cause without the missing final narrative. The recorded symptom was: %s%s
### Affected Resources
- %s (%s)
### Recommendation
%s
### Conclusion
NEEDS_ATTENTION: %s No resolution is being inferred.`,
finding.Title,
resourceName,
successCount,
checkWord,
toolList,
errorNote,
symptom,
classification,
resourceName,
resourceDetails,
recommendation,
proposalConclusion,
)
}
// fallbackToolDisplayName resolves an operator-facing tool name for the
// automatic fallback summary. It prefers the real tool name from the assistant's
// tool call (idToName), and only falls back to the tool-use id when that id
// itself reads like a tool name — never an opaque provider call id
// (`call_…`, `toolu_…`, `fc_…`), which must not leak into chat-visible text.
func fallbackToolDisplayName(toolUseID string, idToName map[string]string) string {
if name := strings.TrimSpace(idToName[strings.TrimSpace(toolUseID)]); name != "" {
return displayToolName(name)
}
normalized := normalizeToolUseID(toolUseID)
lower := strings.ToLower(normalized)
if normalized == "" ||
strings.HasPrefix(lower, "call_") ||
strings.HasPrefix(lower, "toolu_") ||
strings.HasPrefix(lower, "fc_") {
return ""
}
return displayToolName(normalized)
}
func displayToolName(name string) string {
return strings.TrimPrefix(strings.TrimSpace(name), "pulse_")
}
func normalizeToolUseID(toolUseID string) string {
toolUseID = strings.TrimSpace(toolUseID)
if toolUseID == "" {
return ""
}
lastUnderscore := strings.LastIndex(toolUseID, "_")
if lastUnderscore <= 0 || lastUnderscore >= len(toolUseID)-1 {
return toolUseID
}
suffix := toolUseID[lastUnderscore+1:]
for _, r := range suffix {
if r < '0' || r > '9' {
return toolUseID
}
}
return toolUseID[:lastUnderscore]
}