Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package chat
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"sync"
"testing"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/agentcapabilities"
"github.com/rcourtman/pulse-go-rewrite/internal/agentexec"
"github.com/rcourtman/pulse-go-rewrite/internal/ai/providers"
"github.com/rcourtman/pulse-go-rewrite/internal/ai/tools"
)
type policyBlockedEvidenceAgentServer struct{}
func (policyBlockedEvidenceAgentServer) GetConnectedAgents() []agentexec.ConnectedAgent {
return []agentexec.ConnectedAgent{{AgentID: "agent-1", Hostname: "host-1"}}
}
func (policyBlockedEvidenceAgentServer) ExecuteCommand(context.Context, string, agentexec.ExecuteCommandPayload) (*agentexec.CommandResultPayload, error) {
return &agentexec.CommandResultPayload{
Stdout: agentcapabilities.PolicyBlockedToolMarker("uptime", "blocked by policy"),
}, nil
}
func TestRequiresOrderedPatrolFindingLifecycleExecution(t *testing.T) {
tests := []struct {
name string
calls []providers.ToolCall
want bool
}{
{name: "independent reads stay parallel", calls: []providers.ToolCall{{Name: agentcapabilities.PulseQueryToolName}, {Name: agentcapabilities.PulseReadToolName}}},
{name: "independent finding reports stay parallel", calls: []providers.ToolCall{{Name: agentcapabilities.PatrolReportFindingToolName}, {Name: agentcapabilities.PatrolReportFindingToolName}}},
{name: "findings read before assessment is ordered", calls: []providers.ToolCall{{Name: agentcapabilities.PatrolGetFindingsToolName}, {Name: agentcapabilities.PatrolAssessFindingToolName}}, want: true},
{name: "findings read before report is ordered", calls: []providers.ToolCall{{Name: agentcapabilities.PatrolGetFindingsToolName}, {Name: agentcapabilities.PatrolReportFindingToolName}}, want: true},
{name: "provider order remains relevant even when reversed", calls: []providers.ToolCall{{Name: agentcapabilities.PatrolResolveFindingToolName}, {Name: agentcapabilities.PatrolGetFindingsToolName}}, want: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := requiresOrderedPatrolFindingLifecycleExecution(tt.calls); got != tt.want {
t.Fatalf("requiresOrderedPatrolFindingLifecycleExecution() = %v, want %v", got, tt.want)
}
})
}
}
func TestWithoutProviderToolRemovesOnlyNamedCapability(t *testing.T) {
available := []providers.Tool{
{Name: agentcapabilities.PatrolGetFindingsToolName},
{Name: agentcapabilities.PatrolReportFindingToolName},
{Name: "pulse_query"},
}
filtered := withoutProviderTool(available, agentcapabilities.PatrolGetFindingsToolName)
if len(filtered) != 2 || filtered[0].Name != agentcapabilities.PatrolReportFindingToolName || filtered[1].Name != "pulse_query" {
t.Fatalf("filtered tools = %+v", filtered)
}
if len(available) != 3 {
t.Fatalf("source tool projection was mutated: %+v", available)
}
}
func TestAgenticLoop_Setters(t *testing.T) {
loop := &AgenticLoop{}
loop.SetMaxTurns(7)
if loop.maxTurns != 7 {
t.Fatalf("expected maxTurns=7, got %d", loop.maxTurns)
}
loop.SetMaxEvidenceCalls(5)
if loop.maxEvidenceCalls != 5 {
t.Fatalf("expected maxEvidenceCalls=5, got %d", loop.maxEvidenceCalls)
}
loop.SetProviderInfo("provider", "model")
if loop.providerName != "provider" || loop.modelName != "model" {
t.Fatalf("expected provider/model to be set")
}
called := false
loop.SetBudgetChecker(func() error {
called = true
return nil
})
if loop.budgetChecker == nil {
t.Fatalf("expected budgetChecker to be set")
}
_ = loop.budgetChecker()
if !called {
t.Fatalf("expected budgetChecker to be invoked")
}
loop.SetExecutionProfile(tools.ProfilePatrolDetection)
if loop.streamIdleTimeout != patrolProviderStreamIdleTimeout {
t.Fatalf("detection stream idle timeout=%s want %s", loop.streamIdleTimeout, patrolProviderStreamIdleTimeout)
}
loop.SetExecutionProfile(tools.ProfilePatrolInvestigation)
if loop.streamIdleTimeout != patrolProviderStreamIdleTimeout {
t.Fatalf("investigation stream idle timeout=%s want %s", loop.streamIdleTimeout, patrolProviderStreamIdleTimeout)
}
loop.SetExecutionProfile(tools.ProfileInteractiveAssistant)
if loop.streamIdleTimeout != 0 {
t.Fatalf("interactive stream idle timeout=%s want provider default", loop.streamIdleTimeout)
}
}
func TestPatrolDetectionInferenceAllowanceDoesNotConstrainInvestigation(t *testing.T) {
detection := providers.ChatRequest{}
applyExecutionInferenceAllowance(&detection, tools.ProfilePatrolDetection, false, 0)
if detection.MaxTokens != patrolDetectionTurnOutputAllowance || detection.ReasoningEffort != providers.ReasoningEffortLow {
t.Fatalf("detection allowance = %+v", detection)
}
nearLimit := providers.ChatRequest{}
applyExecutionInferenceAllowance(&nearLimit, tools.ProfilePatrolDetection, true, patrolDetectionRunOutputAllowance-400)
if nearLimit.MaxTokens != patrolDetectionMinimumAllowance || nearLimit.ReasoningEffort != providers.ReasoningEffortLow {
t.Fatalf("near-limit allowance = %+v", nearLimit)
}
investigation := providers.ChatRequest{MaxTokens: 4096, ReasoningEffort: providers.ReasoningEffortHigh}
applyExecutionInferenceAllowance(&investigation, tools.ProfilePatrolInvestigation, false, patrolDetectionRunOutputAllowance)
if investigation.MaxTokens != 4096 || investigation.ReasoningEffort != providers.ReasoningEffortHigh {
t.Fatalf("investigation allowance was changed: %+v", investigation)
}
}
func TestInvestigationEvidenceBudgetHelpers(t *testing.T) {
available := []providers.Tool{
{Name: agentcapabilities.PulseQueryToolName},
{Name: agentcapabilities.PulseAlertsToolName},
{Name: agentcapabilities.PulseKnowledgeToolName},
{Name: agentcapabilities.PulseSummarizeToolName},
{Name: agentcapabilities.PatrolGetFindingsToolName},
{Name: agentcapabilities.PatrolProposeActionToolName},
{Name: agentcapabilities.PatrolActionCapabilitiesToolName},
}
terminal := investigationTerminalTools(available, true)
if len(terminal) != 1 || terminal[0].Name != agentcapabilities.PatrolProposeActionToolName {
t.Fatalf("terminal tools = %+v, want only patrol_propose_action", terminal)
}
withConclusion := append(available, providers.Tool{Name: agentcapabilities.PatrolConcludeInvestigationToolName})
terminal = investigationTerminalTools(withConclusion, true)
if len(terminal) != 2 || terminal[0].Name != agentcapabilities.PatrolProposeActionToolName || terminal[1].Name != agentcapabilities.PatrolConcludeInvestigationToolName {
t.Fatalf("terminal tools = %+v, want patrol_propose_action then patrol_conclude_investigation", terminal)
}
terminal = investigationTerminalTools(withConclusion, false)
if len(terminal) != 1 || terminal[0].Name != agentcapabilities.PatrolConcludeInvestigationToolName {
t.Fatalf("final-turn terminal tools = %+v, want only patrol_conclude_investigation", terminal)
}
if isInvestigationEvidenceTool(agentcapabilities.PatrolConcludeInvestigationToolName) {
t.Fatal("recording a conclusion must not consume evidence budget")
}
if !isInvestigationEvidenceTool(agentcapabilities.PulseQueryToolName) {
t.Fatal("read-only query must consume evidence budget")
}
if isInvestigationEvidenceTool(agentcapabilities.PatrolProposeActionToolName) {
t.Fatal("terminal proposal must not consume evidence budget")
}
if isInvestigationEvidenceTool(agentcapabilities.PatrolActionCapabilitiesToolName) {
t.Fatal("capability lookup must not count as infrastructure evidence")
}
for _, name := range []string{
agentcapabilities.PulseAlertsToolName,
agentcapabilities.PulseKnowledgeToolName,
agentcapabilities.PulseSummarizeToolName,
agentcapabilities.PatrolGetFindingsToolName,
} {
if isInvestigationEvidenceTool(name) {
t.Fatalf("derived context tool %q must not count as infrastructure evidence", name)
}
}
}
func TestAgenticLoopPatrolInvestigationRejectsUnadvertisedToolBeforeExecution(t *testing.T) {
provider := &stubStreamingProvider{}
var requests []providers.ChatRequest
turn := 0
provider.chatStream = func(_ context.Context, req providers.ChatRequest, callback providers.StreamCallback) error {
requests = append(requests, req)
turn++
switch turn {
case 1:
callback(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{ToolCalls: []providers.ToolCall{{ID: "invented", Name: "container_state", Input: map[string]interface{}{"resource_id": "container-1"}}}}})
case 2:
callback(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{ToolCalls: []providers.ToolCall{{ID: "evidence", Name: agentcapabilities.PulseQueryToolName, Input: map[string]interface{}{"action": "search", "query": "containers"}}}}})
default:
callback(providers.StreamEvent{Type: "content", Data: providers.ContentEvent{Text: "grounded conclusion"}})
callback(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{}})
}
return nil
}
executor := tools.NewPulseToolExecutor(tools.ExecutorConfig{StateProvider: &mockStateProvider{}})
executor.ApplyExecutionProfile(tools.ProfilePatrolInvestigation)
loop := NewAgenticLoop(provider, executor, "system")
loop.SetExecutionProfile(tools.ProfilePatrolInvestigation)
loop.SetMaxTurns(4)
var rejected ToolEndData
result, err := loop.ExecuteWithTools(
context.Background(),
"unknown-tool-repair",
[]Message{{Role: "user", Content: "investigate"}},
[]providers.Tool{{Name: agentcapabilities.PulseQueryToolName}, {Name: agentcapabilities.PatrolProposeActionToolName}},
func(event StreamEvent) {
if event.Type != "tool_end" {
return
}
var candidate ToolEndData
if json.Unmarshal(event.Data, &candidate) == nil && candidate.ID == "invented" {
rejected = candidate
}
},
)
if err != nil {
t.Fatalf("ExecuteWithTools: %v", err)
}
if rejected.Success || !strings.Contains(rejected.Output, "TOOL_NOT_ADVERTISED") {
t.Fatalf("unknown tool result = %+v, want exact-manifest rejection before execution", rejected)
}
if !strings.Contains(rejected.Output, agentcapabilities.PulseQueryToolName) || !strings.Contains(rejected.Output, agentcapabilities.PatrolProposeActionToolName) {
t.Fatalf("unknown tool correction did not use exact turn manifest: %q", rejected.Output)
}
if loop.GetTotalEvidenceCalls() != 1 {
t.Fatalf("evidence calls = %d, want only the advertised call counted", loop.GetTotalEvidenceCalls())
}
if len(requests) != 3 || len(result) == 0 || result[len(result)-1].Content != "grounded conclusion" {
t.Fatalf("repair did not reach grounded conclusion: requests=%d result=%+v", len(requests), result)
}
}
func TestAgenticLoopPatrolInvestigationEnforcesEvidenceBudget(t *testing.T) {
provider := &stubStreamingProvider{}
var requests []providers.ChatRequest
turn := 0
provider.chatStream = func(_ context.Context, req providers.ChatRequest, callback providers.StreamCallback) error {
requests = append(requests, req)
turn++
switch turn {
case 1:
callback(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{ToolCalls: []providers.ToolCall{{ID: "evidence-1", Name: agentcapabilities.PulseQueryToolName, Input: map[string]interface{}{"action": "search", "query": "containers"}}}}})
case 2:
callback(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{ToolCalls: []providers.ToolCall{{ID: "evidence-2", Name: agentcapabilities.PulseReadToolName, Input: map[string]interface{}{"operation": "resource"}}}}})
default:
callback(providers.StreamEvent{Type: "content", Data: providers.ContentEvent{Text: "final investigation summary"}})
callback(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{}})
}
return nil
}
executor := tools.NewPulseToolExecutor(tools.ExecutorConfig{StateProvider: &mockStateProvider{}})
executor.ApplyExecutionProfile(tools.ProfilePatrolInvestigation)
loop := NewAgenticLoop(provider, executor, "system")
loop.SetExecutionProfile(tools.ProfilePatrolInvestigation)
loop.SetMaxEvidenceCalls(1)
loop.SetMaxTurns(4)
available := []providers.Tool{
{Name: agentcapabilities.PulseQueryToolName},
{Name: agentcapabilities.PulseReadToolName},
{Name: agentcapabilities.PatrolProposeActionToolName},
}
result, err := loop.ExecuteWithTools(context.Background(), "evidence-budget", []Message{{Role: "user", Content: "investigate"}}, available, func(StreamEvent) {})
if err != nil {
t.Fatalf("ExecuteWithTools: %v", err)
}
if loop.GetTotalEvidenceCalls() != 1 {
t.Fatalf("evidence calls = %d, want 1", loop.GetTotalEvidenceCalls())
}
if loop.GetTotalToolCalls() != 2 {
t.Fatalf("model-selected tool calls = %d, want 2", loop.GetTotalToolCalls())
}
if loop.GetTotalModelTurns() != 3 {
t.Fatalf("model turns = %d, want 3", loop.GetTotalModelTurns())
}
if len(requests) != 3 {
t.Fatalf("provider requests = %d, want 3", len(requests))
}
if len(requests[2].Tools) != 1 || requests[2].Tools[0].Name != agentcapabilities.PatrolProposeActionToolName {
t.Fatalf("post-budget tools = %+v, want only patrol_propose_action", requests[2].Tools)
}
if !strings.Contains(requests[2].System, "Evidence-tool calls remaining within this run's configured limit: 0.") {
t.Fatalf("post-budget system prompt missing completion contract: %q", requests[2].System)
}
if len(result) == 0 || !strings.Contains(result[len(result)-1].Content, "final investigation summary") {
t.Fatalf("final result = %+v", result)
}
}
func TestIsPatrolFindingLifecycleWrite(t *testing.T) {
for _, toolName := range []string{
agentcapabilities.PatrolReportFindingToolName,
agentcapabilities.PatrolAssessFindingToolName,
agentcapabilities.PatrolResolveFindingToolName,
} {
if !isPatrolFindingLifecycleWrite(toolName) {
t.Fatalf("expected %s to be a Patrol finding lifecycle write", toolName)
}
if kind := ClassifyToolCall(toolName, nil); kind != ToolKindWrite {
t.Fatalf("%s must retain governed write classification, got %s", toolName, kind)
}
}
for _, toolName := range []string{
agentcapabilities.PatrolGetFindingsToolName,
agentcapabilities.PulseControlToolName,
agentcapabilities.PulseQueryToolName,
} {
if isPatrolFindingLifecycleWrite(toolName) {
t.Fatalf("did not expect %s to mutate finding lifecycle state", toolName)
}
}
}
func TestPatrolFinalFindingDecisionRequestNarrowsWatchTools(t *testing.T) {
req := providers.ChatRequest{
System: "full Patrol prompt",
ToolChoice: &providers.ToolChoice{Type: providers.ToolChoiceRequired},
}
available := []providers.Tool{
{Name: agentcapabilities.PulseQueryToolName},
{Name: agentcapabilities.PatrolGetFindingsToolName},
{Name: agentcapabilities.PatrolReportFindingToolName},
{Name: agentcapabilities.PatrolAssessFindingToolName},
{Name: agentcapabilities.PatrolResolveFindingToolName},
}
if !applyPatrolFinalFindingDecisionRequest(&req, tools.ProfilePatrolDetection, available) {
t.Fatal("expected Watch detection to retain a final finding decision turn")
}
if req.System != patrolFinalFindingDecisionSystemPrompt {
t.Fatalf("final decision request retained full system prompt: %q", req.System)
}
if !strings.Contains(req.System, strings.Join(tools.PatrolReportFindingRequiredArguments(), ", ")) {
t.Fatalf("final decision prompt omitted required report arguments: %q", req.System)
}
if req.ToolChoice != nil {
t.Fatalf("final decision request must remain model-owned, got choice %+v", req.ToolChoice)
}
if len(req.Tools) != 2 || req.Tools[0].Name != agentcapabilities.PatrolReportFindingToolName || req.Tools[1].Name != agentcapabilities.PatrolAssessFindingToolName {
t.Fatalf("final decision tools = %+v, want report and assess only", req.Tools)
}
unchanged := providers.ChatRequest{System: "investigation", Tools: available}
if applyPatrolFinalFindingDecisionRequest(&unchanged, tools.ProfilePatrolInvestigation, available) {
t.Fatal("investigation profile must not gain finding write tools")
}
if unchanged.System != "investigation" || len(unchanged.Tools) != len(available) {
t.Fatalf("inapplicable helper mutated investigation request: %+v", unchanged)
}
}
func TestFilterRepeatedPatrolFindingLifecycleCallsPreservesDistinctDecisions(t *testing.T) {
first := providers.ToolCall{ID: "report-a", Name: agentcapabilities.PatrolReportFindingToolName, Input: map[string]interface{}{
"key": "health-a", "resource_id": "app-container-a",
}}
repeated := first
repeated.ID = "report-a-repeated"
second := providers.ToolCall{ID: "report-b", Name: agentcapabilities.PatrolReportFindingToolName, Input: map[string]interface{}{
"key": "health-b", "resource_id": "app-container-b",
}}
read := providers.ToolCall{ID: "query", Name: agentcapabilities.PulseQueryToolName, Input: map[string]interface{}{"action": "fleet"}}
filtered, suppressed := filterRepeatedPatrolFindingLifecycleCalls([]providers.ToolCall{first, repeated, second, read}, nil)
if len(filtered) != 3 || filtered[0].ID != first.ID || filtered[1].ID != second.ID || filtered[2].ID != read.ID {
t.Fatalf("filtered calls = %+v, want first report, distinct report, and read", filtered)
}
if len(suppressed) != 1 || suppressed[0].ID != repeated.ID {
t.Fatalf("suppressed calls = %+v, want exact repeated report only", suppressed)
}
filtered, suppressed = filterRepeatedPatrolFindingLifecycleCalls([]providers.ToolCall{first, second}, map[string]struct{}{
toolCallKey(first.Name, first.Input): {},
})
if len(filtered) != 1 || filtered[0].ID != second.ID || len(suppressed) != 1 || suppressed[0].ID != first.ID {
t.Fatalf("accepted-call filtering = filtered:%+v suppressed:%+v", filtered, suppressed)
}
}
func TestAgenticLoopRemovesPatrolFindingsReadAfterSuccess(t *testing.T) {
provider := &stubStreamingProvider{}
executor := tools.NewPulseToolExecutor(tools.ExecutorConfig{})
executor.ApplyExecutionProfile(tools.ProfilePatrolDetection)
creator := &repairTestPatrolFindingCreator{}
executor.SetPatrolFindingCreator(creator)
loop := NewAgenticLoop(provider, executor, "full Patrol prompt")
loop.SetExecutionProfile(tools.ProfilePatrolDetection)
loop.SetMaxTurns(4)
providerCalls := 0
provider.chatStream = func(_ context.Context, req providers.ChatRequest, callback providers.StreamCallback) error {
providerCalls++
switch providerCalls {
case 1:
if !hasProviderTool(req.Tools, agentcapabilities.PatrolGetFindingsToolName) {
t.Fatalf("initial request omitted active-findings snapshot: %+v", req.Tools)
}
call := providers.ToolCall{ID: "get-findings", Name: agentcapabilities.PatrolGetFindingsToolName, Input: map[string]interface{}{}}
callback(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{ToolCalls: []providers.ToolCall{call}}})
case 2:
if hasProviderTool(req.Tools, agentcapabilities.PatrolGetFindingsToolName) {
t.Fatalf("successful one-shot active-findings read remained available: %+v", req.Tools)
}
if !hasProviderTool(req.Tools, agentcapabilities.PatrolReportFindingToolName) {
t.Fatalf("unrelated Watch capabilities were removed: %+v", req.Tools)
}
callback(providers.StreamEvent{Type: "content", Data: providers.ContentEvent{Text: "All clear."}})
callback(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{}})
default:
t.Fatalf("unexpected provider call %d", providerCalls)
}
return nil
}
available := []providers.Tool{
{Name: agentcapabilities.PatrolGetFindingsToolName},
{Name: agentcapabilities.PatrolReportFindingToolName},
}
result, err := loop.ExecuteWithTools(context.Background(), "one-shot-findings-read", []Message{{Role: "user", Content: "check this healthy resource"}}, available, func(StreamEvent) {})
if err != nil {
t.Fatalf("Watch run failed: %v", err)
}
if providerCalls != 2 || len(result) == 0 || result[len(result)-1].Content != "All clear." {
t.Fatalf("provider calls=%d result=%+v", providerCalls, result)
}
}
func TestAgenticLoopUsesCorePatrolFindingSnapshotOnFirstTurn(t *testing.T) {
provider := &stubStreamingProvider{}
executor := tools.NewPulseToolExecutor(tools.ExecutorConfig{})
executor.ApplyExecutionProfile(tools.ProfilePatrolDetection)
creator := &repairTestPatrolFindingCreator{}
creator.GetActiveFindings("", "")
executor.SetPatrolFindingCreator(creator)
loop := NewAgenticLoop(provider, executor, "full Patrol prompt")
loop.SetExecutionProfile(tools.ProfilePatrolDetection)
loop.SetMaxTurns(3)
providerCalls := 0
provider.chatStream = func(_ context.Context, req providers.ChatRequest, callback providers.StreamCallback) error {
providerCalls++
switch providerCalls {
case 1:
if hasProviderTool(req.Tools, agentcapabilities.PatrolGetFindingsToolName) {
t.Fatalf("core-owned snapshot left patrol_get_findings on the first provider request: %+v", req.Tools)
}
if !hasProviderTool(req.Tools, agentcapabilities.PatrolReportFindingToolName) {
t.Fatalf("core-owned snapshot removed finding report authority: %+v", req.Tools)
}
call := providers.ToolCall{ID: "report-restart", Name: agentcapabilities.PatrolReportFindingToolName, Input: map[string]interface{}{
"key": "restart-loop",
"severity": "warning",
"category": "reliability",
"resource_id": "app-container-api",
"resource_name": "api",
"resource_type": "app-container",
"title": "API repeatedly restarts",
"description": "Current state confirms repeated container exits.",
"recommendation": "Inspect the container exit reason and verify stable uptime after correction.",
"evidence": "Provider state reports six restarts and the container currently restarting.",
}}
callback(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{ToolCalls: []providers.ToolCall{call}}})
case 2:
callback(providers.StreamEvent{Type: "content", Data: providers.ContentEvent{Text: "One reliability finding reported."}})
callback(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{}})
default:
t.Fatalf("unexpected provider call %d", providerCalls)
}
return nil
}
available := []providers.Tool{
{Name: agentcapabilities.PatrolGetFindingsToolName},
{Name: agentcapabilities.PatrolReportFindingToolName},
}
result, err := loop.ExecuteWithTools(context.Background(), "core-findings-snapshot", []Message{{Role: "user", Content: "assess the restart evidence"}}, available, func(StreamEvent) {})
if err != nil {
t.Fatalf("Watch run failed: %v", err)
}
if providerCalls != 2 || len(creator.created) != 1 {
t.Fatalf("provider calls=%d created=%+v, want one accepted report and one summary turn", providerCalls, creator.created)
}
if len(result) == 0 || result[len(result)-1].Content != "One reliability finding reported." {
t.Fatalf("unexpected result: %+v", result)
}
}
func TestAgenticLoopRecoversTruncatedPatrolDecisionWithGovernedFindingTurn(t *testing.T) {
provider := &stubStreamingProvider{}
executor := tools.NewPulseToolExecutor(tools.ExecutorConfig{})
executor.ApplyExecutionProfile(tools.ProfilePatrolDetection)
creator := &repairTestPatrolFindingCreator{}
creator.GetActiveFindings("", "")
executor.SetPatrolFindingCreator(creator)
loop := NewAgenticLoop(provider, executor, "full Patrol prompt")
loop.SetExecutionProfile(tools.ProfilePatrolDetection)
loop.SetMaxTurns(3)
providerCalls := 0
var recoveryRequest providers.ChatRequest
provider.chatStream = func(_ context.Context, req providers.ChatRequest, callback providers.StreamCallback) error {
providerCalls++
switch providerCalls {
case 1:
callback(providers.StreamEvent{Type: "content", Data: providers.ContentEvent{Text: "The unhealthy container is a confirmed incident. I should report it, but first I will restate all of my reasoning..."}})
callback(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{StopReason: "length", InputTokens: 1000, OutputTokens: patrolDetectionTurnOutputAllowance}})
case 2:
recoveryRequest = req
call := providers.ToolCall{ID: "report-health", Name: agentcapabilities.PatrolReportFindingToolName, Input: map[string]interface{}{
"key": "container-health-failed",
"severity": "warning",
"category": "reliability",
"resource_id": "app-container-api",
"resource_name": "api",
"resource_type": "app-container",
"title": "API health check is failing",
"description": "The container is running but its current health check is unhealthy.",
"recommendation": "Inspect the health endpoint and recent container logs.",
"evidence": "Provider state reports the running container as unhealthy.",
}}
callback(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{StopReason: "tool_use", ToolCalls: []providers.ToolCall{call}, InputTokens: 900, OutputTokens: 200}})
case 3:
callback(providers.StreamEvent{Type: "content", Data: providers.ContentEvent{Text: "One reliability finding reported."}})
callback(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{StopReason: "end_turn", InputTokens: 500, OutputTokens: 20}})
default:
t.Fatalf("unexpected provider call %d", providerCalls)
}
return nil
}
available := []providers.Tool{
{Name: agentcapabilities.PulseQueryToolName},
{Name: agentcapabilities.PatrolReportFindingToolName},
{Name: agentcapabilities.PatrolAssessFindingToolName},
}
result, err := loop.ExecuteWithTools(context.Background(), "truncated-watch-decision", []Message{{Role: "user", Content: "Assess the unhealthy container."}}, available, func(StreamEvent) {})
if err != nil {
t.Fatalf("Watch recovery failed: %v", err)
}
if providerCalls != 3 || len(creator.created) != 1 {
t.Fatalf("provider calls=%d created=%+v, want one recovery report and one summary", providerCalls, creator.created)
}
if len(recoveryRequest.Tools) != 2 || hasProviderTool(recoveryRequest.Tools, agentcapabilities.PulseQueryToolName) {
t.Fatalf("recovery tools = %+v, want only finding decisions", recoveryRequest.Tools)
}
if !strings.Contains(recoveryRequest.System, "previous model turn exhausted its output budget") {
t.Fatalf("recovery prompt = %q", recoveryRequest.System)
}
if recoveryRequest.MaxTokens != patrolDetectionRecoveryAllowance {
t.Fatalf("recovery MaxTokens = %d, want %d within the existing run budget", recoveryRequest.MaxTokens, patrolDetectionRecoveryAllowance)
}
var persisted strings.Builder
for _, message := range result {
if message.Role == "assistant" {
persisted.WriteString(message.Content)
}
}
if strings.Contains(persisted.String(), "restate all of my reasoning") || !strings.Contains(persisted.String(), "One reliability finding reported.") {
t.Fatalf("persisted Patrol analysis = %q", persisted.String())
}
}
type objectiveRecoveryProposer struct {
calls []tools.PatrolObserverProposalInput
}
func (p *objectiveRecoveryProposer) ProposeObserver(input tools.PatrolObserverProposalInput) (tools.PatrolObserverProposalResult, error) {
p.calls = append(p.calls, input)
return tools.PatrolObserverProposalResult{
ObjectiveID: input.ObjectiveID, Revision: input.ExpectedRevision + 1,
ObserverID: "observer-recovered", Version: 1, State: "proposed",
ArtifactDigest: "sha256:recovered", CoverageState: "uncovered", CoverageReason: "observer_proposed",
}, nil
}
func TestAgenticLoopRecoversTruncatedObjectiveMissionWithProposalOnlyTurn(t *testing.T) {
provider := &stubStreamingProvider{}
executor := tools.NewPulseToolExecutor(tools.ExecutorConfig{})
executor.ApplyExecutionProfile(tools.ProfilePatrolDetection)
proposer := &objectiveRecoveryProposer{}
executor.SetPatrolObserverProposer(proposer)
loop := NewAgenticLoop(provider, executor, "objective mission")
loop.SetExecutionProfile(tools.ProfilePatrolDetection)
loop.SetMaxTurns(3)
providerCalls := 0
var recoveryRequest providers.ChatRequest
var summaryRequest providers.ChatRequest
provider.chatStream = func(_ context.Context, req providers.ChatRequest, callback providers.StreamCallback) error {
providerCalls++
switch providerCalls {
case 1:
callback(providers.StreamEvent{Type: "content", Data: providers.ContentEvent{Text: "I will carefully consider every possible observer design before deciding..."}})
callback(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{StopReason: "length", InputTokens: 1000, OutputTokens: patrolDetectionTurnOutputAllowance}})
case 2:
recoveryRequest = req
call := providers.ToolCall{ID: "propose-observer", Name: agentcapabilities.PatrolProposeObserverToolName, Input: map[string]interface{}{
"objective_id": "objective-1", "expected_revision": float64(1),
"evidence_fit": "proxy", "interpretation": "Use reachability as a wake signal", "trigger_kind": "interval",
"probe_json": `{"runtime":"pulse-availability-state/v1","target_id":"availability-1","path":"probe_outcome","operator":"equals","value":"reachable","sample_interval_seconds":30,"wake_after_consecutive_failures":2}`,
"wake_evidence": "The endpoint is unreachable twice", "requirements_json": `{}`,
}}
callback(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{StopReason: "tool_use", ToolCalls: []providers.ToolCall{call}, InputTokens: 800, OutputTokens: 180}})
case 3:
summaryRequest = req
callback(providers.StreamEvent{Type: "content", Data: providers.ContentEvent{Text: "A bounded proxy observer was proposed for core validation."}})
callback(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{StopReason: "end_turn", InputTokens: 400, OutputTokens: 20}})
default:
t.Fatalf("unexpected provider call %d", providerCalls)
}
return nil
}
available := []providers.Tool{{Name: agentcapabilities.PatrolProposeObserverToolName}}
result, err := loop.ExecuteWithTools(context.Background(), "truncated-objective-mission", []Message{{Role: "user", Content: "Objective objective-1 revision 1."}}, available, func(StreamEvent) {})
if err != nil {
t.Fatalf("objective recovery failed: %v", err)
}
if providerCalls != 3 || len(proposer.calls) != 1 {
t.Fatalf("provider calls=%d proposals=%d, want one bounded recovery proposal", providerCalls, len(proposer.calls))
}
if len(recoveryRequest.Tools) != 1 || recoveryRequest.Tools[0].Name != agentcapabilities.PatrolProposeObserverToolName {
t.Fatalf("recovery tools = %+v, want proposal only", recoveryRequest.Tools)
}
if recoveryRequest.ToolChoice == nil || recoveryRequest.ToolChoice.Type != providers.ToolChoiceRequired {
t.Fatalf("recovery tool choice = %+v, want required", recoveryRequest.ToolChoice)
}
if !strings.Contains(recoveryRequest.System, "previous model turn exhausted its output budget") || strings.Contains(recoveryRequest.System, "finding") {
t.Fatalf("recovery prompt = %q", recoveryRequest.System)
}
if recoveryRequest.MaxTokens != patrolDetectionRecoveryAllowance {
t.Fatalf("recovery MaxTokens = %d, want %d", recoveryRequest.MaxTokens, patrolDetectionRecoveryAllowance)
}
if len(summaryRequest.Tools) != 0 {
t.Fatalf("post-proposal tools = %+v, want prose-only completion", summaryRequest.Tools)
}
var persisted strings.Builder
for _, message := range result {
if message.Role == "assistant" {
persisted.WriteString(message.Content)
}
}
if strings.Contains(persisted.String(), "carefully consider") || !strings.Contains(persisted.String(), "bounded proxy observer") {
t.Fatalf("persisted objective result = %q", persisted.String())
}
}
func TestPatrolObjectiveOutputLimitRecoveryRejectsMixedWatchTools(t *testing.T) {
req := providers.ChatRequest{System: "normal Watch"}
available := []providers.Tool{
{Name: agentcapabilities.PatrolProposeObserverToolName},
{Name: agentcapabilities.PatrolReportFindingToolName},
{Name: agentcapabilities.PatrolAssessFindingToolName},
}
if applyPatrolObjectiveOutputLimitRecoveryRequest(&req, tools.ProfilePatrolDetection, available) {
t.Fatal("normal Watch projection was misclassified as an objective-only mission")
}
if req.System != "normal Watch" || req.ToolChoice != nil {
t.Fatalf("rejected mixed projection mutated request: %+v", req)
}
}
func TestAgenticLoopFailsClosedAfterRepeatedTruncatedObjectiveRecovery(t *testing.T) {
provider := &stubStreamingProvider{}
executor := tools.NewPulseToolExecutor(tools.ExecutorConfig{})
executor.ApplyExecutionProfile(tools.ProfilePatrolDetection)
loop := NewAgenticLoop(provider, executor, "objective mission")
loop.SetExecutionProfile(tools.ProfilePatrolDetection)
loop.SetMaxTurns(1)
providerCalls := 0
provider.chatStream = func(_ context.Context, req providers.ChatRequest, callback providers.StreamCallback) error {
providerCalls++
if providerCalls == 2 {
if len(req.Tools) != 1 || req.Tools[0].Name != agentcapabilities.PatrolProposeObserverToolName || req.ToolChoice == nil {
t.Fatalf("second request was not bounded objective recovery: %+v", req)
}
}
callback(providers.StreamEvent{Type: "content", Data: providers.ContentEvent{Text: "unfinished"}})
callback(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{StopReason: map[int]string{1: "length", 2: "max_tokens"}[providerCalls], OutputTokens: patrolDetectionTurnOutputAllowance}})
return nil
}
result, err := loop.ExecuteWithTools(context.Background(), "twice-truncated-objective", []Message{{Role: "user", Content: "Objective objective-1 revision 1."}}, []providers.Tool{{Name: agentcapabilities.PatrolProposeObserverToolName}}, func(StreamEvent) {})
if err == nil || !strings.Contains(err.Error(), "objective observer proposal") {
t.Fatalf("error = %v, want objective-specific fail-closed output-budget error", err)
}
if providerCalls != 2 {
t.Fatalf("provider calls = %d, want one bounded retry", providerCalls)
}
for _, message := range result {
if message.Role == "assistant" && strings.TrimSpace(message.Content) != "" {
t.Fatalf("truncated content leaked into objective result: %+v", result)
}
}
}
func TestAgenticLoopFailsClosedAfterRepeatedTruncatedPatrolDecision(t *testing.T) {
provider := &stubStreamingProvider{}
executor := tools.NewPulseToolExecutor(tools.ExecutorConfig{})
executor.ApplyExecutionProfile(tools.ProfilePatrolDetection)
loop := NewAgenticLoop(provider, executor, "full Patrol prompt")
loop.SetExecutionProfile(tools.ProfilePatrolDetection)
loop.SetMaxTurns(1)
providerCalls := 0
provider.chatStream = func(_ context.Context, req providers.ChatRequest, callback providers.StreamCallback) error {
providerCalls++
if providerCalls == 2 {
if len(req.Tools) != 2 || !strings.Contains(req.System, "output budget") {
t.Fatalf("second request was not bounded decision recovery: %+v", req)
}
}
callback(providers.StreamEvent{Type: "content", Data: providers.ContentEvent{Text: "unfinished"}})
callback(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{StopReason: map[int]string{1: "length", 2: "max_tokens"}[providerCalls], OutputTokens: patrolDetectionTurnOutputAllowance}})
return nil
}
available := []providers.Tool{
{Name: agentcapabilities.PatrolReportFindingToolName},
{Name: agentcapabilities.PatrolAssessFindingToolName},
}
result, err := loop.ExecuteWithTools(context.Background(), "twice-truncated-watch-decision", []Message{{Role: "user", Content: "Assess the resource."}}, available, func(StreamEvent) {})
if err == nil || !strings.Contains(err.Error(), "exhausted its output budget twice") {
t.Fatalf("error = %v, want fail-closed output-budget error", err)
}
if providerCalls != 2 {
t.Fatalf("provider calls = %d, want one bounded retry", providerCalls)
}
for _, message := range result {
if message.Role == "assistant" && strings.TrimSpace(message.Content) != "" {
t.Fatalf("truncated content leaked into Patrol result: %+v", result)
}
}
}
func TestAgenticLoop_FinalWatchTurnRetainsOnlyFindingDecisions(t *testing.T) {
provider := &stubStreamingProvider{}
loop := NewAgenticLoop(provider, nil, "full Patrol prompt")
loop.SetExecutionProfile(tools.ProfilePatrolDetection)
loop.SetMaxTurns(1)
provider.chatStream = func(ctx context.Context, req providers.ChatRequest, callback providers.StreamCallback) error {
if req.System != patrolFinalFindingDecisionSystemPrompt {
t.Fatalf("final Watch system prompt = %q", req.System)
}
if len(req.Tools) != 2 || req.Tools[0].Name != agentcapabilities.PatrolReportFindingToolName || req.Tools[1].Name != agentcapabilities.PatrolAssessFindingToolName {
t.Fatalf("final Watch tools = %+v, want report and assess only", req.Tools)
}
callback(providers.StreamEvent{Type: "content", Data: providers.ContentEvent{Text: "All clear."}})
callback(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{InputTokens: 1, OutputTokens: 1}})
return nil
}
result, err := loop.ExecuteWithTools(
context.Background(),
"final-watch-decision",
[]Message{{Role: "user", Content: "current scoped state"}},
[]providers.Tool{
{Name: agentcapabilities.PulseQueryToolName},
{Name: agentcapabilities.PatrolReportFindingToolName},
{Name: agentcapabilities.PatrolAssessFindingToolName},
},
func(event StreamEvent) {},
)
if err != nil {
t.Fatalf("ExecuteWithTools: %v", err)
}
if len(result) != 1 || result[0].Content != "All clear." {
t.Fatalf("final Watch result = %+v", result)
}
}
func TestPruneMessagesForModel_Stateless(t *testing.T) {
prev := StatelessContext
StatelessContext = true
defer func() { StatelessContext = prev }()
messages := []Message{
{Role: "user", Content: "first"},
{Role: "assistant", Content: "ok"},
{Role: "user", Content: "second"},
{Role: "assistant", Content: "done"},
}
pruned := pruneMessagesForModel(messages)
if len(pruned) != 1 {
t.Fatalf("expected 1 message, got %d", len(pruned))
}
if pruned[0].Content != "second" {
t.Fatalf("expected last user message to be kept")
}
}
func TestPruneMessagesForModel_SkipsOrphanedToolResults(t *testing.T) {
// Build a message list longer than MaxContextMessagesLimit so pruning occurs.
messages := make([]Message, 0, MaxContextMessagesLimit+2)
messages = append(messages, Message{Role: "user", Content: "a"})
messages = append(messages, Message{Role: "assistant", Content: "b"})
// This tool result should be dropped if it becomes the first item after pruning.
messages = append(messages, Message{Role: "assistant", ToolResult: &ToolResult{Content: "tool"}})
for i := 0; i < MaxContextMessagesLimit; i++ {
messages = append(messages, Message{Role: "user", Content: "msg"})
}
pruned := pruneMessagesForModel(messages)
if len(pruned) == 0 {
t.Fatalf("expected pruned messages")
}
if pruned[0].ToolResult != nil {
t.Fatalf("expected leading tool result to be pruned")
}
}
func TestPruneMessagesForModel_SkipsAssistantWithToolCalls(t *testing.T) {
// Ensure assistant tool calls at the start of the pruned window are skipped.
messages := make([]Message, 0, MaxContextMessagesLimit+3)
messages = append(messages, Message{Role: "user", Content: "seed"})
messages = append(messages, Message{Role: "assistant", Content: "seed"})
messages = append(messages, Message{Role: "assistant", ToolCalls: []ToolCall{{Name: "pulse_query"}}})
messages = append(messages, Message{Role: "assistant", ToolResult: &ToolResult{Content: "result"}})
for i := 0; i < MaxContextMessagesLimit; i++ {
messages = append(messages, Message{Role: "user", Content: "msg"})
}
pruned := pruneMessagesForModel(messages)
if len(pruned) == 0 {
t.Fatalf("expected pruned messages")
}
if pruned[0].Role == "assistant" && len(pruned[0].ToolCalls) > 0 {
t.Fatalf("expected assistant tool-call message to be pruned")
}
if pruned[0].ToolResult != nil {
t.Fatalf("expected tool result following pruned tool call to be removed")
}
}
type stubStreamingProvider struct {
lastRequest providers.ChatRequest
chatStream func(ctx context.Context, req providers.ChatRequest, callback providers.StreamCallback) error
}
type repairTestPatrolFindingCreator struct {
checked bool
complete bool
created []tools.PatrolFindingInput
}
type orderedPatrolFindingCreator struct {
mu sync.Mutex
checked bool
assessments []tools.PatrolFindingAssessmentInput
}
func (c *orderedPatrolFindingCreator) CreateFinding(tools.PatrolFindingInput) (string, bool, error) {
return "finding-existing", false, nil
}
func (c *orderedPatrolFindingCreator) ResolveFinding(string, string) error { return nil }
func (c *orderedPatrolFindingCreator) GetActiveFindings(string, string) []tools.PatrolFindingInfo {
// Make the historical parallel-execution race deterministic: an assessment
// started alongside this read observes checked=false and is rejected.
time.Sleep(25 * time.Millisecond)
c.mu.Lock()
c.checked = true
c.mu.Unlock()
return []tools.PatrolFindingInfo{{ID: "finding-existing", Severity: "warning", Category: "reliability", ResourceID: "app-container-existing", ResourceName: "existing", ResourceType: "app-container", Title: "Container unhealthy"}}
}
func (c *orderedPatrolFindingCreator) HasCheckedFindings() bool {
c.mu.Lock()
defer c.mu.Unlock()
return c.checked
}
func (c *orderedPatrolFindingCreator) AssessFinding(input tools.PatrolFindingAssessmentInput) error {
c.mu.Lock()
defer c.mu.Unlock()
c.assessments = append(c.assessments, input)
return nil
}
func (c *repairTestPatrolFindingCreator) CreateFinding(input tools.PatrolFindingInput) (string, bool, error) {
c.created = append(c.created, input)
return "finding-" + input.ResourceID, true, nil
}
func (c *repairTestPatrolFindingCreator) ResolveFinding(string, string) error { return nil }
func (c *repairTestPatrolFindingCreator) GetActiveFindings(resourceID, minSeverity string) []tools.PatrolFindingInfo {
c.checked = true
if strings.TrimSpace(resourceID) == "" {
minimum := strings.ToLower(strings.TrimSpace(minSeverity))
c.complete = minimum == "" || minimum == "info"
}
return nil
}
func (c *repairTestPatrolFindingCreator) HasCheckedFindings() bool { return c.checked }
func (c *repairTestPatrolFindingCreator) HasCompleteFindingSnapshot() bool { return c.complete }
func (s *stubStreamingProvider) Chat(ctx context.Context, req providers.ChatRequest) (*providers.ChatResponse, error) {
return &providers.ChatResponse{Content: "ok"}, nil
}
func (s *stubStreamingProvider) ChatStream(ctx context.Context, req providers.ChatRequest, callback providers.StreamCallback) error {
s.lastRequest = req
if s.chatStream != nil {
return s.chatStream(ctx, req, callback)
}
callback(providers.StreamEvent{Type: "content", Data: providers.ContentEvent{Text: "summary"}})
callback(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{InputTokens: 1, OutputTokens: 1}})
return nil
}
func (s *stubStreamingProvider) SupportsThinking(model string) bool { return false }
func (s *stubStreamingProvider) TestConnection(ctx context.Context) error { return nil }
func (s *stubStreamingProvider) Name() string { return "stub" }
func (s *stubStreamingProvider) ListModels(ctx context.Context) ([]providers.ModelInfo, error) {
return nil, nil
}
func TestAgenticLoopOrdersSameTurnPatrolFindingReadBeforeAssessment(t *testing.T) {
provider := &stubStreamingProvider{}
executor := tools.NewPulseToolExecutor(tools.ExecutorConfig{})
executor.ApplyExecutionProfile(tools.ProfilePatrolDetection)
creator := &orderedPatrolFindingCreator{}
executor.SetPatrolFindingCreator(creator)
loop := NewAgenticLoop(provider, executor, "full Patrol prompt")
loop.SetExecutionProfile(tools.ProfilePatrolDetection)
loop.SetMaxTurns(2)
providerCalls := 0
provider.chatStream = func(_ context.Context, req providers.ChatRequest, callback providers.StreamCallback) error {
providerCalls++
if req.MaxTokens <= 0 || req.ReasoningEffort != providers.ReasoningEffortLow {
t.Fatalf("Watch request %d missing inference allowance: %+v", providerCalls, req)
}
switch providerCalls {
case 1:
callback(providers.StreamEvent{Type: "content", Data: providers.ContentEvent{Text: "intermediate routing prose that must not become the run summary"}})
getCall := providers.ToolCall{ID: "get-findings", Name: agentcapabilities.PatrolGetFindingsToolName, Input: map[string]interface{}{}}
assessCall := providers.ToolCall{ID: "assess-finding", Name: agentcapabilities.PatrolAssessFindingToolName, Input: map[string]interface{}{
"finding_id": "finding-existing",
"verdict": "present",
"evidence": "Current provider state remains unhealthy.",
"reason": "The current health check still fails.",
}}
for _, call := range []providers.ToolCall{getCall, assessCall} {
callback(providers.StreamEvent{Type: "tool_start", Data: providers.ToolStartEvent{ID: call.ID, Name: call.Name, Input: call.Input}})
}
callback(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{ToolCalls: []providers.ToolCall{getCall, assessCall}}})
case 2:
callback(providers.StreamEvent{Type: "content", Data: providers.ContentEvent{Text: "Finding remains present."}})
callback(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{}})
default:
t.Fatalf("unexpected provider call %d", providerCalls)
}
return nil
}
available := []providers.Tool{
{Name: agentcapabilities.PatrolGetFindingsToolName},
{Name: agentcapabilities.PatrolAssessFindingToolName},
}
var toolEnds []ToolEndData
result, err := loop.ExecuteWithTools(context.Background(), "ordered-finding-lifecycle", []Message{{Role: "user", Content: "Reassess the active finding."}}, available, func(event StreamEvent) {
if event.Type != "tool_end" {
return
}
var data ToolEndData
if err := json.Unmarshal(event.Data, &data); err != nil {
t.Fatalf("decode tool_end: %v", err)
}
toolEnds = append(toolEnds, data)
})
if err != nil {
t.Fatalf("ordered Patrol finding lifecycle failed: %v", err)
}
if providerCalls != 2 {
t.Fatalf("provider calls = %d, want lifecycle turn and summary", providerCalls)
}
var persistedText strings.Builder
for _, message := range result {
if message.Role == "assistant" {
persistedText.WriteString(message.Content)
}
}
if strings.Contains(persistedText.String(), "intermediate routing prose") || !strings.Contains(persistedText.String(), "Finding remains present.") {
t.Fatalf("persisted Patrol analysis = %q", persistedText.String())
}
if len(toolEnds) != 2 || !toolEnds[0].Success || !toolEnds[1].Success {
t.Fatalf("tool results = %+v, want ordered successful read and assessment", toolEnds)
}
creator.mu.Lock()
defer creator.mu.Unlock()
if len(creator.assessments) != 1 || creator.assessments[0].FindingID != "finding-existing" {
t.Fatalf("assessments = %+v, want one existing-finding verdict", creator.assessments)
}
}
func TestAgenticLoopRepairsRejectedSiblingAfterMixedPatrolFindingBatch(t *testing.T) {
provider := &stubStreamingProvider{}
executor := tools.NewPulseToolExecutor(tools.ExecutorConfig{})
executor.ApplyExecutionProfile(tools.ProfilePatrolDetection)
creator := &repairTestPatrolFindingCreator{}
executor.SetPatrolFindingCreator(creator)
loop := NewAgenticLoop(provider, executor, "full Patrol prompt")
loop.SetExecutionProfile(tools.ProfilePatrolDetection)
loop.SetMaxTurns(5)
completeReport := func(resourceID, resourceName, title string) map[string]interface{} {
return map[string]interface{}{
"key": "container-health-failed",
"severity": "warning",
"category": "reliability",
"resource_id": resourceID,
"resource_name": resourceName,
"resource_type": "app-container",
"title": title,
"description": "Container is running but its health check is unhealthy.",
"recommendation": "Inspect the health endpoint and recent logs.",
"evidence": "Provider state reports running and unhealthy with zero restarts.",
}
}
var repairRequest providers.ChatRequest
var originalSystem string
providerCalls := 0
provider.chatStream = func(_ context.Context, req providers.ChatRequest, callback providers.StreamCallback) error {
providerCalls++
switch providerCalls {
case 1:
originalSystem = strings.SplitN(req.System, "\nCURRENT TIME:", 2)[0]
if !strings.HasPrefix(originalSystem, "full Patrol prompt") {
t.Fatalf("initial request lost original prompt: %q", originalSystem)
}
call := providers.ToolCall{ID: "get-findings", Name: agentcapabilities.PatrolGetFindingsToolName, Input: map[string]interface{}{}}
callback(providers.StreamEvent{Type: "tool_start", Data: providers.ToolStartEvent{ID: call.ID, Name: call.Name, Input: call.Input}})
callback(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{ToolCalls: []providers.ToolCall{call}}})
case 2:
if !strings.HasPrefix(req.System, originalSystem+"\nCURRENT TIME:") {
t.Fatalf("second turn changed the original investigation prompt: %q", req.System)
}
accepted := providers.ToolCall{ID: "report-api", Name: agentcapabilities.PatrolReportFindingToolName, Input: completeReport("app-container-api", "api", "API health check failing")}
rejectedInput := completeReport("app-container-worker", "worker", "Worker health check failing")
delete(rejectedInput, "title")
rejected := providers.ToolCall{ID: "report-worker-invalid", Name: agentcapabilities.PatrolReportFindingToolName, Input: rejectedInput}
for _, call := range []providers.ToolCall{accepted, rejected} {
callback(providers.StreamEvent{Type: "tool_start", Data: providers.ToolStartEvent{ID: call.ID, Name: call.Name, Input: call.Input}})
}
callback(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{ToolCalls: []providers.ToolCall{accepted, rejected}}})
case 3:
repairRequest = req
repaired := providers.ToolCall{ID: "report-worker-repaired", Name: agentcapabilities.PatrolReportFindingToolName, Input: completeReport("app-container-worker", "worker", "Worker health check failing")}
callback(providers.StreamEvent{Type: "tool_start", Data: providers.ToolStartEvent{ID: repaired.ID, Name: repaired.Name, Input: repaired.Input}})
callback(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{ToolCalls: []providers.ToolCall{repaired}}})
case 4:
if !strings.HasPrefix(req.System, originalSystem+"\nCURRENT TIME:") || !providerToolIsAdvertised(req.Tools, agentcapabilities.PulseQueryToolName) {
t.Fatalf("post-repair summary request = %+v", req)
}
callback(providers.StreamEvent{Type: "content", Data: providers.ContentEvent{Text: "Two findings reported."}})
callback(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{}})
default:
t.Fatalf("unexpected provider call %d", providerCalls)
}
return nil
}
available := []providers.Tool{
{Name: agentcapabilities.PulseQueryToolName},
{Name: agentcapabilities.PatrolGetFindingsToolName},
{Name: agentcapabilities.PatrolReportFindingToolName},
{Name: agentcapabilities.PatrolAssessFindingToolName},
{Name: agentcapabilities.PatrolResolveFindingToolName},
}
var toolEnds []ToolEndData
result, err := loop.ExecuteWithTools(context.Background(), "mixed-finding-repair", []Message{{Role: "user", Content: "check both containers"}}, available, func(event StreamEvent) {
if event.Type != "tool_end" {
return
}
var data ToolEndData
if err := json.Unmarshal(event.Data, &data); err != nil {
t.Fatalf("decode tool_end: %v", err)
}
toolEnds = append(toolEnds, data)
})
if err != nil {
t.Fatalf("mixed finding repair failed: %v", err)
}
if providerCalls != 4 {
t.Fatalf("provider calls = %d, want get/findings, mixed batch, repair, summary; tool ends = %+v; created = %+v", providerCalls, toolEnds, creator.created)
}
if !strings.HasPrefix(repairRequest.System, originalSystem+"\nCURRENT TIME:") {
t.Fatalf("repair system prompt = %q", repairRequest.System)
}
if !providerToolIsAdvertised(repairRequest.Tools, agentcapabilities.PulseQueryToolName) {
t.Fatalf("repair request tools = %+v, want evidence tools retained", repairRequest.Tools)
}
if len(creator.created) != 2 || creator.created[0].ResourceID != "app-container-api" || creator.created[1].ResourceID != "app-container-worker" {
t.Fatalf("created findings = %+v, want each resource exactly once", creator.created)
}
failedCalls := 0
for _, event := range toolEnds {
if !event.Success {
failedCalls++
}
}
if failedCalls != 1 {
t.Fatalf("failed tool calls = %d, want original rejected sibling preserved", failedCalls)
}
if len(result) == 0 || result[len(result)-1].Content != "Two findings reported." {
t.Fatalf("final result = %+v", result)
}
}
func TestAgenticLoopAllowsEvidenceReadsBetweenIndependentFindings(t *testing.T) {
provider := &stubStreamingProvider{}
executor := tools.NewPulseToolExecutor(tools.ExecutorConfig{StateProvider: &mockStateProvider{}})
executor.ApplyExecutionProfile(tools.ProfilePatrolDetection)
creator := &repairTestPatrolFindingCreator{}
executor.SetPatrolFindingCreator(creator)
loop := NewAgenticLoop(provider, executor, "full Patrol prompt")
loop.SetExecutionProfile(tools.ProfilePatrolDetection)
loop.SetMaxTurns(6)
completeReport := func(resourceID, resourceName, title string) map[string]interface{} {
return map[string]interface{}{
"key": "container-health-failed",
"severity": "warning",
"category": "reliability",
"resource_id": resourceID,
"resource_name": resourceName,
"resource_type": "app-container",
"title": title,
"description": "Container is running but its health check is unhealthy.",
"recommendation": "Inspect the health endpoint and recent logs.",
"evidence": "Provider state reports running and unhealthy with zero restarts.",
}
}
var originalSystem string
providerCalls := 0
provider.chatStream = func(_ context.Context, req providers.ChatRequest, callback providers.StreamCallback) error {
providerCalls++
switch providerCalls {
case 1:
originalSystem = strings.SplitN(req.System, "\nCURRENT TIME:", 2)[0]
if !strings.HasPrefix(originalSystem, "full Patrol prompt") {
t.Fatalf("initial request lost original prompt: %q", originalSystem)
}
call := providers.ToolCall{ID: "get-findings", Name: agentcapabilities.PatrolGetFindingsToolName, Input: map[string]interface{}{}}
callback(providers.StreamEvent{Type: "tool_start", Data: providers.ToolStartEvent{ID: call.ID, Name: call.Name, Input: call.Input}})
callback(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{ToolCalls: []providers.ToolCall{call}}})
case 2:
call := providers.ToolCall{ID: "report-api", Name: agentcapabilities.PatrolReportFindingToolName, Input: completeReport("app-container-api", "api", "API health check failing")}
callback(providers.StreamEvent{Type: "tool_start", Data: providers.ToolStartEvent{ID: call.ID, Name: call.Name, Input: call.Input}})
callback(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{ToolCalls: []providers.ToolCall{call}}})
case 3:
if !strings.HasPrefix(req.System, originalSystem+"\nCURRENT TIME:") || !providerToolIsAdvertised(req.Tools, agentcapabilities.PulseQueryToolName) {
t.Fatalf("recording a finding curtailed investigation: %+v", req)
}
call := providers.ToolCall{ID: "read-worker", Name: agentcapabilities.PulseQueryToolName, Input: map[string]interface{}{"action": "search", "query": "worker"}}
callback(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{ToolCalls: []providers.ToolCall{call}}})
case 4:
if !strings.HasPrefix(req.System, originalSystem+"\nCURRENT TIME:") {
t.Fatalf("first completion request system = %q", req.System)
}
last := req.Messages[len(req.Messages)-1].ToolResult
if last == nil || last.ToolUseID != "read-worker" || last.IsError {
t.Fatalf("post-report evidence missing or failed: %+v", last)
}
call := providers.ToolCall{ID: "report-worker", Name: agentcapabilities.PatrolReportFindingToolName, Input: completeReport("app-container-worker", "worker", "Worker health check failing")}
callback(providers.StreamEvent{Type: "tool_start", Data: providers.ToolStartEvent{ID: call.ID, Name: call.Name, Input: call.Input}})
callback(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{ToolCalls: []providers.ToolCall{call}}})
case 5:
if !strings.HasPrefix(req.System, originalSystem+"\nCURRENT TIME:") {
t.Fatalf("second completion request system = %q", req.System)
}
if !providerToolIsAdvertised(req.Tools, agentcapabilities.PulseQueryToolName) {
t.Fatalf("second completion request tools = %+v", req.Tools)
}
callback(providers.StreamEvent{Type: "content", Data: providers.ContentEvent{Text: "Two independent unhealthy containers were recorded."}})
callback(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{}})
default:
t.Fatalf("unexpected provider call %d", providerCalls)
}
return nil
}
available := []providers.Tool{
{Name: agentcapabilities.PulseQueryToolName},
{Name: agentcapabilities.PatrolGetFindingsToolName},
{Name: agentcapabilities.PatrolReportFindingToolName},
{Name: agentcapabilities.PatrolAssessFindingToolName},
}
result, err := loop.ExecuteWithTools(context.Background(), "sequential-independent-findings", []Message{{Role: "user", Content: "check both containers"}}, available, func(StreamEvent) {})
if err != nil {
t.Fatalf("sequential independent finding run failed: %v", err)
}
if providerCalls != 5 {
t.Fatalf("provider calls = %d, want snapshot, first report, evidence read, second report, summary", providerCalls)
}
if len(creator.created) != 2 || creator.created[0].ResourceID != "app-container-api" || creator.created[1].ResourceID != "app-container-worker" {
t.Fatalf("created findings = %+v, want both independent resources exactly once", creator.created)
}
if len(result) == 0 || result[len(result)-1].Content != "Two independent unhealthy containers were recorded." {
t.Fatalf("final result = %+v", result)
}
}
func TestAgenticLoopPreservesAcceptedFindingAndProviderFailure(t *testing.T) {
provider := &stubStreamingProvider{}
executor := tools.NewPulseToolExecutor(tools.ExecutorConfig{})
executor.ApplyExecutionProfile(tools.ProfilePatrolDetection)
creator := &repairTestPatrolFindingCreator{}
executor.SetPatrolFindingCreator(creator)
loop := NewAgenticLoop(provider, executor, "full Patrol prompt")
loop.SetExecutionProfile(tools.ProfilePatrolDetection)
loop.SetMaxTurns(4)
reportInput := map[string]interface{}{
"key": "container-health-failed",
"severity": "warning",
"category": "reliability",
"resource_id": "app-container-api",
"resource_name": "api",
"resource_type": "app-container",
"title": "API health check failing",
"description": "The API container is running but its health check is unhealthy.",
"recommendation": "Inspect the health endpoint and recent logs.",
"evidence": "Provider state reports running and unhealthy with zero restarts.",
}
providerCalls := 0
provider.chatStream = func(_ context.Context, req providers.ChatRequest, callback providers.StreamCallback) error {
providerCalls++
switch providerCalls {
case 1:
call := providers.ToolCall{ID: "get-findings", Name: agentcapabilities.PatrolGetFindingsToolName, Input: map[string]interface{}{}}
callback(providers.StreamEvent{Type: "tool_start", Data: providers.ToolStartEvent{ID: call.ID, Name: call.Name, Input: call.Input}})
callback(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{ToolCalls: []providers.ToolCall{call}}})
case 2:
call := providers.ToolCall{ID: "report-api", Name: agentcapabilities.PatrolReportFindingToolName, Input: reportInput}
callback(providers.StreamEvent{Type: "tool_start", Data: providers.ToolStartEvent{ID: call.ID, Name: call.Name, Input: call.Input}})
callback(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{ToolCalls: []providers.ToolCall{call}}})
case 3:
return errors.New("provider failed after recording the finding")
default:
t.Fatalf("unexpected provider call %d", providerCalls)
}
return nil
}
available := []providers.Tool{
{Name: agentcapabilities.PatrolGetFindingsToolName},
{Name: agentcapabilities.PatrolReportFindingToolName},
{Name: agentcapabilities.PatrolAssessFindingToolName},
}
result, err := loop.ExecuteWithTools(context.Background(), "failed-watch-continuation", []Message{{Role: "user", Content: "check both containers"}}, available, func(StreamEvent) {})
if err == nil || !strings.Contains(err.Error(), "provider failed after recording the finding") {
t.Fatalf("provider failure was hidden after an accepted finding: %v", err)
}
if providerCalls != 3 {
t.Fatalf("provider calls = %d, want read, report, failed provider turn", providerCalls)
}
if len(creator.created) != 1 || creator.created[0].ResourceID != "app-container-api" {
t.Fatalf("created findings = %+v, want accepted API finding preserved", creator.created)
}
for _, message := range result {
if message.Role == "assistant" && message.Content != "" {
t.Fatalf("failed run invented terminal prose: %q", message.Content)
}
}
}
func TestAgenticLoopSuppressesExactRepeatedAcceptedPatrolReport(t *testing.T) {
provider := &stubStreamingProvider{}
executor := tools.NewPulseToolExecutor(tools.ExecutorConfig{})
executor.ApplyExecutionProfile(tools.ProfilePatrolDetection)
creator := &repairTestPatrolFindingCreator{}
executor.SetPatrolFindingCreator(creator)
loop := NewAgenticLoop(provider, executor, "full Patrol prompt")
loop.SetExecutionProfile(tools.ProfilePatrolDetection)
loop.SetMaxTurns(4)
reportInput := map[string]interface{}{
"key": "restart-loop",
"severity": "warning",
"category": "reliability",
"resource_id": "app-container-worker",
"resource_name": "worker",
"resource_type": "app-container",
"title": "Container restart count exceeds threshold",
"description": "Provider reported four restarts while the container remained running.",
"recommendation": "Inspect the restart pattern and recent exit logs.",
"evidence": "Provider state reports four restarts.",
}
var originalSystem string
providerCalls := 0
provider.chatStream = func(_ context.Context, req providers.ChatRequest, callback providers.StreamCallback) error {
providerCalls++
switch providerCalls {
case 1:
originalSystem = strings.SplitN(req.System, "\nCURRENT TIME:", 2)[0]
if !strings.HasPrefix(originalSystem, "full Patrol prompt") {
t.Fatalf("initial request lost original prompt: %q", originalSystem)
}
call := providers.ToolCall{ID: "get-findings", Name: agentcapabilities.PatrolGetFindingsToolName, Input: map[string]interface{}{}}
callback(providers.StreamEvent{Type: "tool_start", Data: providers.ToolStartEvent{ID: call.ID, Name: call.Name, Input: call.Input}})
callback(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{ToolCalls: []providers.ToolCall{call}}})
case 2:
call := providers.ToolCall{ID: "report-first", Name: agentcapabilities.PatrolReportFindingToolName, Input: reportInput}
callback(providers.StreamEvent{Type: "tool_start", Data: providers.ToolStartEvent{ID: call.ID, Name: call.Name, Input: call.Input}})
callback(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{ToolCalls: []providers.ToolCall{call}}})
case 3:
if !strings.HasPrefix(req.System, originalSystem+"\nCURRENT TIME:") {
t.Fatalf("repeat request system = %q, want original prompt", req.System)
}
call := providers.ToolCall{ID: "report-repeated", Name: agentcapabilities.PatrolReportFindingToolName, Input: reportInput}
callback(providers.StreamEvent{Type: "tool_start", Data: providers.ToolStartEvent{ID: call.ID, Name: call.Name, Input: call.Input}})
callback(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{ToolCalls: []providers.ToolCall{call}}})
case 4:
if len(req.Tools) != 0 {
t.Fatalf("final summary unexpectedly offered tools: %+v", req.Tools)
}
callback(providers.StreamEvent{Type: "content", Data: providers.ContentEvent{Text: "The restart issue was recorded once."}})
callback(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{}})
default:
t.Fatalf("unexpected provider call %d", providerCalls)
}
return nil
}
available := []providers.Tool{
{Name: agentcapabilities.PatrolGetFindingsToolName},
{Name: agentcapabilities.PatrolReportFindingToolName},
{Name: agentcapabilities.PatrolAssessFindingToolName},
}
var reportStarts, reportEnds int
result, err := loop.ExecuteWithTools(context.Background(), "repeated-accepted-report", []Message{{Role: "user", Content: "check worker"}}, available, func(event StreamEvent) {
switch event.Type {
case "tool_start":
var data ToolStartData
if err := json.Unmarshal(event.Data, &data); err != nil {
t.Fatalf("decode tool_start: %v", err)
}
if data.Name == agentcapabilities.PatrolReportFindingToolName {
reportStarts++
}
case "tool_end":
var data ToolEndData
if err := json.Unmarshal(event.Data, &data); err != nil {
t.Fatalf("decode tool_end: %v", err)
}
if data.Name == agentcapabilities.PatrolReportFindingToolName {
reportEnds++
}
}
})
if err != nil {
t.Fatalf("repeated accepted report run failed: %v", err)
}
if providerCalls != 4 {
t.Fatalf("provider calls = %d, want read, report, absorbed repeat, summary", providerCalls)
}
if reportStarts != 1 || reportEnds != 1 {
t.Fatalf("canonical report events = starts:%d ends:%d, want exactly one pair", reportStarts, reportEnds)
}
if len(creator.created) != 1 || creator.created[0].ResourceID != "app-container-worker" {
t.Fatalf("created findings = %+v, want one canonical report", creator.created)
}
if loop.GetTotalToolCalls() != 2 {
t.Fatalf("effective tool calls = %d, want findings read plus one report", loop.GetTotalToolCalls())
}
if len(result) == 0 || result[len(result)-1].Content != "The restart issue was recorded once." {
t.Fatalf("final result = %+v", result)
}
}
func TestEnsureFinalTextResponse(t *testing.T) {
provider := &stubStreamingProvider{}
loop := &AgenticLoop{provider: provider, baseSystemPrompt: "prompt"}
result, summaryErr := loop.ensureFinalTextResponse(
context.Background(),
"session-1",
[]Message{{Role: "assistant", Content: ""}},
[]providers.Message{{Role: "assistant", Content: ""}},
func(event StreamEvent) {},
)
if summaryErr != nil {
t.Fatal(summaryErr)
}
if len(result) != 2 {
t.Fatalf("expected summary message to be appended")
}
if provider.lastRequest.ToolChoice != nil || len(provider.lastRequest.Tools) != 0 {
t.Fatalf("expected summary call to omit tools and tool_choice, got tools=%d choice=%+v", len(provider.lastRequest.Tools), provider.lastRequest.ToolChoice)
}
if loop.GetTotalModelTurns() != 1 {
t.Fatalf("successful final summary model turns = %d, want 1", loop.GetTotalModelTurns())
}
provider.chatStream = func(ctx context.Context, req providers.ChatRequest, callback providers.StreamCallback) error {
return errors.New("boom")
}
result2, summaryErr2 := loop.ensureFinalTextResponse(
context.Background(),
"session-2",
[]Message{
{Role: "assistant", Content: ""},
{Role: "user", ToolResult: &ToolResult{ToolUseID: "pulse_query_0", Content: "{\"nodes\":1}", IsError: false}},
},
[]providers.Message{{Role: "assistant", Content: ""}},
func(event StreamEvent) {},
)
if summaryErr2 != nil {
t.Fatal(summaryErr2)
}
if len(result2) != 3 {
t.Fatalf("expected fallback summary message when provider errors")
}
if !strings.Contains(result2[len(result2)-1].Content, "didn't return a written summary") {
t.Fatalf("expected deterministic fallback summary, got %q", result2[len(result2)-1].Content)
}
if loop.GetTotalModelTurns() != 1 {
t.Fatalf("failed final summary changed model turns to %d", loop.GetTotalModelTurns())
}
provider.chatStream = func(ctx context.Context, req providers.ChatRequest, callback providers.StreamCallback) error {
callback(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{InputTokens: 1, OutputTokens: 1}})
return nil
}
result3, summaryErr3 := loop.ensureFinalTextResponse(
context.Background(),
"session-3",
[]Message{
{Role: "assistant", Content: ""},
{Role: "user", ToolResult: &ToolResult{ToolUseID: "pulse_metrics_1", Content: "cpu ok", IsError: false}},
},
[]providers.Message{{Role: "assistant", Content: ""}},
func(event StreamEvent) {},
)
if summaryErr3 != nil {
t.Fatal(summaryErr3)
}
if len(result3) != 3 {
t.Fatalf("expected fallback summary when provider returns empty content")
}
fallback3 := result3[len(result3)-1].Content
if !strings.Contains(fallback3, "didn't return a written summary") {
t.Fatalf("expected clean fallback summary, got %q", fallback3)
}
// The fallback must not dump raw tool output / result snippets into the chat.
if strings.Contains(fallback3, "cpu ok") || strings.Contains(fallback3, "result snippet") {
t.Fatalf("fallback summary must not include raw tool output, got %q", fallback3)
}
if loop.GetTotalModelTurns() != 2 {
t.Fatalf("empty completed final summary model turns = %d, want 2 cumulative", loop.GetTotalModelTurns())
}
}
func TestEnsureFinalTextResponseRequiresAssistantTextAfterLatestToolResult(t *testing.T) {
provider := &stubStreamingProvider{}
loop := &AgenticLoop{provider: provider, baseSystemPrompt: "prompt"}
var emitted strings.Builder
result, summaryErr := loop.ensureFinalTextResponse(
context.Background(),
"session-pre-tool-preamble",
[]Message{
{Role: "user", Content: "how many devices are in this?"},
{
Role: "assistant",
Content: "I'll check the device nodes.",
ToolCalls: []ToolCall{
{ID: "call-1", Name: "pulse_read", Input: map[string]interface{}{"command": "ls /dev | wc -l"}},
},
},
{
Role: "user",
ToolResult: &ToolResult{ToolUseID: "pulse_read_0", Content: "4358", IsError: false},
},
{Role: "assistant", Content: ""},
},
[]providers.Message{
{Role: "user", Content: "how many devices are in this?"},
{
Role: "assistant",
Content: "I'll check the device nodes.",
ToolCalls: []providers.ToolCall{
{ID: "call-1", Name: "pulse_read", Input: map[string]interface{}{"command": "ls /dev | wc -l"}},
},
},
{Role: "tool", ToolResult: &providers.ToolResult{ToolUseID: "call-1", Content: "4358"}},
{Role: "assistant", Content: ""},
},
func(event StreamEvent) {
if event.Type != "content" {
return
}
var data ContentData
if err := json.Unmarshal(event.Data, &data); err == nil {
emitted.WriteString(data.Text)
}
},
)
if summaryErr != nil {
t.Fatal(summaryErr)
}
if len(result) != 5 {
t.Fatalf("expected final summary to be appended after tool result, got %d messages", len(result))
}
if result[len(result)-1].Content != "summary" {
t.Fatalf("expected provider summary to be appended, got %q", result[len(result)-1].Content)
}
if emitted.String() != "summary" {
t.Fatalf("expected summary to be streamed, got %q", emitted.String())
}
}
func TestHasFinalAssistantText(t *testing.T) {
tests := []struct {
name string
messages []Message
want bool
}{
{
name: "assistant text without tool results is final",
messages: []Message{{Role: "assistant", Content: "done"}},
want: true,
},
{
name: "assistant text before latest user prompt is not final",
messages: []Message{
{Role: "assistant", Content: "previous answer"},
{Role: "user", Content: "new question"},
},
want: false,
},
{
name: "pre tool text before latest tool result is not final",
messages: []Message{
{Role: "user", Content: "check it"},
{Role: "assistant", Content: "I'll check", ToolCalls: []ToolCall{{Name: "pulse_query"}}},
{Role: "user", ToolResult: &ToolResult{ToolUseID: "pulse_query_0", Content: "ok"}},
},
want: false,
},
{
name: "assistant text after latest tool result is final",
messages: []Message{
{Role: "user", Content: "check it"},
{Role: "assistant", Content: "I'll check", ToolCalls: []ToolCall{{Name: "pulse_query"}}},
{Role: "user", ToolResult: &ToolResult{ToolUseID: "pulse_query_0", Content: "ok"}},
{Role: "assistant", Content: "The result is ok."},
},
want: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := hasFinalAssistantText(tt.messages); got != tt.want {
t.Fatalf("hasFinalAssistantText() = %v, want %v", got, tt.want)
}
})
}
}
func TestEnsureFinalTextResponseAppliesRequestSanitizer(t *testing.T) {
provider := &stubStreamingProvider{}
loop := &AgenticLoop{provider: provider, baseSystemPrompt: "raw-host"}
loop.SetRequestSanitizer(func(req providers.ChatRequest) providers.ChatRequest {
req.System = strings.ReplaceAll(req.System, "raw-host", "[redacted]")
req.Messages = append([]providers.Message(nil), req.Messages...)
for i := range req.Messages {
req.Messages[i].Content = strings.ReplaceAll(req.Messages[i].Content, "raw-host", "[redacted]")
}
return req
})
loop.ensureFinalTextResponse(
context.Background(),
"session-sanitized",
[]Message{{Role: "assistant", Content: ""}},
[]providers.Message{{Role: "user", Content: "check raw-host"}},
func(event StreamEvent) {},
)
if strings.Contains(provider.lastRequest.System, "raw-host") {
t.Fatalf("summary system prompt was not sanitized: %q", provider.lastRequest.System)
}
if strings.Contains(provider.lastRequest.Messages[0].Content, "raw-host") {
t.Fatalf("summary message was not sanitized: %q", provider.lastRequest.Messages[0].Content)
}
}
func TestBuildAutomaticFallbackSummary(t *testing.T) {
summary := buildAutomaticFallbackSummary([]Message{
{Role: "user", ToolResult: &ToolResult{ToolUseID: "pulse_query_0", Content: "nodes ok", IsError: false}},
{Role: "user", ToolResult: &ToolResult{ToolUseID: "pulse_query_1", Content: "containers ok", IsError: false}},
{Role: "user", ToolResult: &ToolResult{ToolUseID: "pulse_read_0", Content: "read failed", IsError: true}},
})
if !strings.Contains(summary, "I ran 2 checks") {
t.Fatalf("unexpected fallback summary: %q", summary)
}
// Tool name is surfaced operator-facing (pulse_ prefix stripped), never the
// raw provider call id.
if !strings.Contains(summary, "query") {
t.Fatalf("expected tool name in fallback summary, got %q", summary)
}
if !strings.Contains(summary, "1 tool error") {
t.Fatalf("expected error count in fallback summary, got %q", summary)
}
}
func TestBuildAutomaticFallbackSummary_UsesToolNamesNotCallIDsOrRawOutput(t *testing.T) {
// Reproduces the real OpenRouter shape that produced the ugly chat dump:
// tool results carry opaque provider call ids, and the real tool name lives
// on the assistant tool call. The fallback must name the tools, not leak the
// call ids, and must not dump raw tool output into chat-visible text.
summary := buildAutomaticFallbackSummary([]Message{
{Role: "assistant", ToolCalls: []ToolCall{
{ID: "call_27f0f389aba4652a1e292dc", Name: "pulse_query"},
{ID: "call_66a4659a104b4ee7807201cb", Name: "pulse_metrics"},
}},
{Role: "user", ToolResult: &ToolResult{ToolUseID: "call_27f0f389aba4652a1e292dc", Content: `{"nodes":2}`, IsError: false}},
{Role: "user", ToolResult: &ToolResult{ToolUseID: "call_66a4659a104b4ee7807201cb", Content: `{"cpuPct":12}`, IsError: false}},
})
if strings.Contains(summary, "call_") {
t.Fatalf("fallback summary must not leak provider call ids, got %q", summary)
}
if !strings.Contains(summary, "query") || !strings.Contains(summary, "metrics") {
t.Fatalf("expected real tool names in fallback summary, got %q", summary)
}
if strings.Contains(summary, "nodes") || strings.Contains(summary, "cpuPct") || strings.Contains(summary, "{") {
t.Fatalf("fallback summary must not dump raw tool output, got %q", summary)
}
}
func TestBuildAutomaticFallbackSummary_InvestigationUsesGroundedReceipt(t *testing.T) {
summary := buildAutomaticFallbackSummary([]Message{
{
Role: "user",
Content: `You are investigating a finding from Pulse Patrol.
## Finding Details
- **Title**: Container health check reported unhealthy while running
- **Severity**: warning
- **Category**: reliability
- **Resource display name**: inventory-api
- **Canonical resource ID**: app-container-123
- **Resource type**: app-container
- **Description**: The container is running, but its health check is unhealthy.
## Evidence Completion Contract`,
},
{Role: "assistant", ToolCalls: []ToolCall{
{ID: "call-query", Name: "pulse_query", Input: map[string]interface{}{"resource_id": "app-container-123"}},
{ID: "call-proposal", Name: "patrol_propose_action", Input: map[string]interface{}{
"resource_id": "app-container-123",
"capability_name": "restart",
}},
}},
{Role: "user", ToolResult: &ToolResult{ToolUseID: "call-query", Content: `{"private_raw_output":"must not leak"}`}},
{Role: "user", ToolResult: &ToolResult{ToolUseID: "call-proposal", Content: `{"ok":true}`}},
})
for _, want := range []string{
"### Investigation Summary",
"health check reported unhealthy",
"### Root Cause",
"inventory-api",
"app-container-123",
"governed restart proposal",
"### Conclusion",
"NEEDS_ATTENTION:",
"No resolution is being inferred",
} {
if !strings.Contains(summary, want) {
t.Fatalf("grounded fallback missing %q:\n%s", want, summary)
}
}
for _, forbidden := range []string{"private_raw_output", "must not leak", "Ask me again"} {
if strings.Contains(summary, forbidden) {
t.Fatalf("grounded fallback leaked or deferred with %q:\n%s", forbidden, summary)
}
}
}
func TestBuildAutomaticFallbackSummary_InvestigationDoesNotTreatFailedProposalAsRecorded(t *testing.T) {
summary := buildAutomaticFallbackSummary([]Message{
{Role: "user", Content: `## Finding Details
- **Title**: Disk is filling up
- **Severity**: warning
- **Category**: capacity
- **Resource display name**: storage-a
- **Canonical resource ID**: storage-123
- **Resource type**: storage
- **Description**: Free space is below the configured threshold.
## Evidence Completion Contract`},
{Role: "assistant", ToolCalls: []ToolCall{{
ID: "call-proposal",
Name: "patrol_propose_action",
Input: map[string]interface{}{
"resource_id": "storage-123",
"capability_name": "delete",
},
}}},
{Role: "user", ToolResult: &ToolResult{ToolUseID: "call-proposal", Content: "policy refused", IsError: true}},
})
if strings.Contains(summary, "governed delete proposal was recorded") {
t.Fatalf("failed proposal must not be presented as recorded:\n%s", summary)
}
if !strings.Contains(summary, "nothing was authorized by this fallback") {
t.Fatalf("failed proposal fallback must stay fail-closed:\n%s", summary)
}
}
func TestExecuteToolSafely_RecoversPanic(t *testing.T) {
exec := tools.NewPulseToolExecutor(tools.ExecutorConfig{})
exec.RegisterTool(tools.RegisteredTool{
Invocation: tools.StaticInvocation(agentcapabilities.ToolCallKindRead, agentcapabilities.MutationNone),
Definition: tools.Tool{
Name: "panic_tool",
InputSchema: tools.InputSchema{
Type: "object",
Properties: map[string]tools.PropertySchema{},
},
},
Handler: func(_ context.Context, _ *tools.PulseToolExecutor, _ map[string]interface{}) (tools.CallToolResult, error) {
panic("boom")
},
})
loop := &AgenticLoop{executor: exec}
result, err := loop.executeToolSafely(context.Background(), "call-1", "panic_tool", map[string]interface{}{})
if err == nil {
t.Fatalf("expected panic recovery error")
}
if !result.IsError {
t.Fatalf("expected error result after panic recovery")
}
if len(result.Content) == 0 || !strings.Contains(result.Content[0].Text, "tool panic in panic_tool") {
t.Fatalf("unexpected panic recovery result: %+v", result)
}
}
func TestAgenticLoop_RetriesProviderStreamBeforeEvents(t *testing.T) {
provider := &stubStreamingProvider{}
executor := tools.NewPulseToolExecutor(tools.ExecutorConfig{})
loop := NewAgenticLoop(provider, executor, "prompt")
callCount := 0
provider.chatStream = func(ctx context.Context, req providers.ChatRequest, callback providers.StreamCallback) error {
callCount++
if callCount == 1 {
return errors.New("connection reset by peer")
}
callback(providers.StreamEvent{Type: "content", Data: providers.ContentEvent{Text: "hello"}})
callback(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{}})
return nil
}
var workflowStates []WorkflowStateData
results, err := loop.Execute(context.Background(), "retry-before-events", []Message{{Role: "user", Content: "hi"}}, func(event StreamEvent) {
if event.Type != "workflow_state" {
return
}
var data WorkflowStateData
if err := json.Unmarshal(event.Data, &data); err != nil {
t.Fatalf("failed to decode workflow_state: %v", err)
}
workflowStates = append(workflowStates, data)
})
if err != nil {
t.Fatalf("expected retry to recover stream failure, got error: %v", err)
}
if callCount != 2 {
t.Fatalf("expected 2 provider attempts, got %d", callCount)
}
if len(workflowStates) != 1 {
t.Fatalf("expected one retry workflow state, got %#v", workflowStates)
}
retry := workflowStates[0]
if retry.Phase != "provider_retry" {
t.Fatalf("retry phase = %q, want provider_retry", retry.Phase)
}
if retry.Attempt != 2 || retry.MaxAttempts != 2 {
t.Fatalf("retry attempts = %d/%d, want 2/2", retry.Attempt, retry.MaxAttempts)
}
if retry.RetryAfterMS != 200 {
t.Fatalf("retry backoff = %dms, want 200ms", retry.RetryAfterMS)
}
if retry.Message != "Selected route connection failed before any output; retrying." {
t.Fatalf("retry message = %q", retry.Message)
}
if len(results) != 1 || results[0].Content != "hello" {
t.Fatalf("unexpected results: %+v", results)
}
}
func TestAgenticLoop_RetriesProviderStreamAfterOnlySuppressedArtifacts(t *testing.T) {
provider := &stubStreamingProvider{}
executor := tools.NewPulseToolExecutor(tools.ExecutorConfig{})
loop := NewAgenticLoop(provider, executor, "prompt")
callCount := 0
provider.chatStream = func(ctx context.Context, req providers.ChatRequest, callback providers.StreamCallback) error {
callCount++
if callCount == 1 {
callback(providers.StreamEvent{
Type: "content",
Data: providers.ContentEvent{
Text: "I'llcheckthedevicenodesinsidethecontainertoanswerthat.Letmecounttheentriesin/devandlisttheblockdevices.",
},
})
callback(providers.StreamEvent{
Type: "content",
Data: providers.ContentEvent{Text: `pulse_read(target_host="current_resource", command="ls /dev | wc -l")`},
})
return errors.New("connection reset by peer")
}
callback(providers.StreamEvent{Type: "content", Data: providers.ContentEvent{Text: "There are 4,358 entries under /dev."}})
callback(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{}})
return nil
}
var emittedContent strings.Builder
var workflowStates []WorkflowStateData
results, err := loop.Execute(context.Background(), "retry-after-suppressed-artifacts", []Message{{Role: "user", Content: "how many devices in this"}}, func(event StreamEvent) {
switch event.Type {
case "content":
var data ContentData
if err := json.Unmarshal(event.Data, &data); err != nil {
t.Fatalf("failed to decode content event: %v", err)
}
emittedContent.WriteString(data.Text)
case "workflow_state":
var data WorkflowStateData
if err := json.Unmarshal(event.Data, &data); err != nil {
t.Fatalf("failed to decode workflow_state: %v", err)
}
workflowStates = append(workflowStates, data)
}
})
if err != nil {
t.Fatalf("expected retry to recover suppressed artifact stream failure, got error: %v", err)
}
if callCount != 2 {
t.Fatalf("expected 2 provider attempts, got %d", callCount)
}
if emittedContent.String() != "There are 4,358 entries under /dev." {
t.Fatalf("unexpected streamed content: %q", emittedContent.String())
}
if strings.Contains(emittedContent.String(), "pulse_read") || strings.Contains(emittedContent.String(), "I'llcheck") {
t.Fatalf("streamed content leaked suppressed artifact text: %q", emittedContent.String())
}
if len(workflowStates) != 1 || workflowStates[0].Phase != "provider_retry" {
t.Fatalf("expected one provider_retry workflow state, got %#v", workflowStates)
}
if len(results) != 1 || results[0].Content != "There are 4,358 entries under /dev." {
t.Fatalf("unexpected results: %+v", results)
}
}
func TestAgenticLoop_DoesNotRetryAfterPartialEvents(t *testing.T) {
provider := &stubStreamingProvider{}
executor := tools.NewPulseToolExecutor(tools.ExecutorConfig{})
loop := NewAgenticLoop(provider, executor, "prompt")
callCount := 0
provider.chatStream = func(ctx context.Context, req providers.ChatRequest, callback providers.StreamCallback) error {
callCount++
callback(providers.StreamEvent{Type: "content", Data: providers.ContentEvent{Text: "partial"}})
return errors.New("connection reset by peer")
}
_, err := loop.Execute(context.Background(), "no-retry-partial", []Message{{Role: "user", Content: "hi"}}, func(event StreamEvent) {})
if err == nil {
t.Fatalf("expected provider error when stream fails after partial output")
}
if callCount != 1 {
t.Fatalf("expected no retry after partial output, got %d attempts", callCount)
}
}
func TestAgenticLoop_RetriesIncompleteNonInteractiveStreamAfterPartialEvent(t *testing.T) {
provider := &stubStreamingProvider{}
executor := tools.NewPulseToolExecutor(tools.ExecutorConfig{})
executor.ApplyExecutionProfile(tools.ProfilePatrolDetection)
loop := NewAgenticLoop(provider, executor, "investigation prompt")
loop.SetExecutionProfile(tools.ProfilePatrolDetection)
callCount := 0
provider.chatStream = func(_ context.Context, _ providers.ChatRequest, callback providers.StreamCallback) error {
callCount++
if callCount == 1 {
callback(providers.StreamEvent{Type: "content", Data: providers.ContentEvent{Text: "partial provider fragment"}})
return errors.New("stream ended before completion marker")
}
callback(providers.StreamEvent{Type: "content", Data: providers.ContentEvent{Text: "complete investigation"}})
callback(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{StopReason: "end_turn"}})
return nil
}
var retries int
result, err := loop.ExecuteWithTools(context.Background(), "retry-incomplete-investigation", []Message{{Role: "user", Content: "investigate"}}, nil, func(event StreamEvent) {
if event.Type == "workflow_state" {
var state WorkflowStateData
if json.Unmarshal(event.Data, &state) == nil && state.Phase == "provider_retry" {
retries++
}
}
})
if err != nil {
t.Fatalf("expected incomplete non-interactive stream replay to recover, got %v", err)
}
if callCount != 2 || retries != 1 {
t.Fatalf("provider calls=%d retries=%d, want 2 calls and one retry", callCount, retries)
}
if len(result) != 1 || result[0].Content != "complete investigation" {
t.Fatalf("durable result = %+v, want only completed replay", result)
}
}
func TestAgenticLoop_RecoversTruncatedInvestigationConclusionFromExistingEvidence(t *testing.T) {
provider := &stubStreamingProvider{}
executor := tools.NewPulseToolExecutor(tools.ExecutorConfig{})
executor.ApplyExecutionProfile(tools.ProfilePatrolInvestigation)
loop := NewAgenticLoop(provider, executor, "full investigation prompt")
loop.SetExecutionProfile(tools.ProfilePatrolInvestigation)
loop.SetMaxTurns(1)
loop.totalEvidenceCalls = 1
providerCalls := 0
var recoveryRequest providers.ChatRequest
provider.chatStream = func(_ context.Context, req providers.ChatRequest, callback providers.StreamCallback) error {
providerCalls++
switch providerCalls {
case 1:
callback(providers.StreamEvent{Type: "content", Data: providers.ContentEvent{Text: "unfinished reasoning"}})
callback(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{StopReason: "length", OutputTokens: 4096}})
case 2:
recoveryRequest = req
callback(providers.StreamEvent{Type: "content", Data: providers.ContentEvent{Text: "### Investigation Summary\nClient is unhealthy.\n\n### Root Cause\nDependency is stopped.\n\n### Affected Resources\nClient.\n\n### Recommendation\nStart dependency.\n\n### Conclusion\nNEEDS_ATTENTION: approval required"}})
callback(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{StopReason: "end_turn", OutputTokens: 120}})
default:
t.Fatalf("unexpected provider call %d", providerCalls)
}
return nil
}
result, err := loop.ExecuteWithTools(context.Background(), "truncated-investigation", []Message{{Role: "user", Content: "investigate the unhealthy client"}}, []providers.Tool{{Name: agentcapabilities.PulseQueryToolName}}, func(StreamEvent) {})
if err != nil {
t.Fatalf("investigation recovery failed: %v", err)
}
if providerCalls != 2 {
t.Fatalf("provider calls=%d, want one bounded recovery", providerCalls)
}
if len(recoveryRequest.Tools) != 0 || recoveryRequest.ToolChoice != nil {
t.Fatalf("recovery request retained tools: %+v", recoveryRequest)
}
if recoveryRequest.System != investigationOutputLimitRecoverySystemPrompt {
t.Fatalf("recovery system prompt = %q", recoveryRequest.System)
}
if recoveryRequest.MaxTokens != investigationOutputLimitRecoveryAllowance || recoveryRequest.ReasoningEffort != providers.ReasoningEffortLow {
t.Fatalf("recovery allowance = %+v", recoveryRequest)
}
for _, message := range recoveryRequest.Messages {
if strings.Contains(message.Content, "unfinished reasoning") || strings.Contains(message.ReasoningContent, "unfinished reasoning") {
t.Fatalf("recovery request retained truncated conclusion: %+v", recoveryRequest.Messages)
}
}
var persisted strings.Builder
for _, message := range result {
if message.Role == "assistant" {
persisted.WriteString(message.Content)
}
}
if strings.Contains(persisted.String(), "unfinished reasoning") || !strings.Contains(persisted.String(), "Dependency is stopped") {
t.Fatalf("persisted investigation = %q", persisted.String())
}
}
func TestAgenticLoop_FailsClosedWhenInvestigationRecoveryAlsoTruncates(t *testing.T) {
provider := &stubStreamingProvider{}
executor := tools.NewPulseToolExecutor(tools.ExecutorConfig{})
executor.ApplyExecutionProfile(tools.ProfilePatrolInvestigation)
loop := NewAgenticLoop(provider, executor, "full investigation prompt")
loop.SetExecutionProfile(tools.ProfilePatrolInvestigation)
loop.SetMaxTurns(1)
providerCalls := 0
provider.chatStream = func(_ context.Context, _ providers.ChatRequest, callback providers.StreamCallback) error {
providerCalls++
callback(providers.StreamEvent{Type: "content", Data: providers.ContentEvent{Text: "still incomplete"}})
callback(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{StopReason: "length", OutputTokens: 4096}})
return nil
}
result, err := loop.ExecuteWithTools(context.Background(), "twice-truncated-investigation", []Message{{Role: "user", Content: "investigate"}}, nil, func(StreamEvent) {})
if err == nil || !strings.Contains(err.Error(), "exhausted its output budget twice") {
t.Fatalf("error = %v, want bounded fail-closed error", err)
}
if providerCalls != 2 {
t.Fatalf("provider calls=%d, want one original and one recovery turn", providerCalls)
}
for _, message := range result {
if strings.TrimSpace(message.Content) != "" {
t.Fatalf("truncated conclusion leaked into durable result: %+v", result)
}
}
}
func TestAgenticLoop_EmitsFallbackErrorEventOnTransportFailure(t *testing.T) {
provider := &stubStreamingProvider{}
executor := tools.NewPulseToolExecutor(tools.ExecutorConfig{})
loop := NewAgenticLoop(provider, executor, "prompt")
provider.chatStream = func(ctx context.Context, req providers.ChatRequest, callback providers.StreamCallback) error {
callback(providers.StreamEvent{Type: "content", Data: providers.ContentEvent{Text: "partial"}})
return errors.New("connection reset by peer")
}
var events []StreamEvent
_, err := loop.Execute(context.Background(), "emit-fallback-error", []Message{{Role: "user", Content: "hi"}}, func(event StreamEvent) {
events = append(events, event)
})
if err == nil {
t.Fatalf("expected provider error when stream fails after partial output")
}
var foundError bool
for _, event := range events {
if event.Type != "error" {
continue
}
foundError = true
var data ErrorData
if decodeErr := json.Unmarshal(event.Data, &data); decodeErr != nil {
t.Fatalf("failed to decode error event payload: %v", decodeErr)
}
if strings.TrimSpace(data.Message) == "" {
t.Fatalf("expected non-empty fallback error message")
}
}
if !foundError {
t.Fatalf("expected fallback error event to be emitted")
}
}
func TestAgenticLoop_IgnoresErrorAfterDoneEvent(t *testing.T) {
provider := &stubStreamingProvider{}
executor := tools.NewPulseToolExecutor(tools.ExecutorConfig{})
loop := NewAgenticLoop(provider, executor, "prompt")
callCount := 0
provider.chatStream = func(ctx context.Context, req providers.ChatRequest, callback providers.StreamCallback) error {
callCount++
callback(providers.StreamEvent{Type: "content", Data: providers.ContentEvent{Text: "complete"}})
callback(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{}})
return errors.New("EOF")
}
results, err := loop.Execute(context.Background(), "ignore-after-done", []Message{{Role: "user", Content: "hi"}}, func(event StreamEvent) {})
if err != nil {
t.Fatalf("expected post-done error to be ignored, got: %v", err)
}
if callCount != 1 {
t.Fatalf("expected single provider attempt, got %d", callCount)
}
if len(results) != 1 || results[0].Content != "complete" {
t.Fatalf("unexpected results: %+v", results)
}
}
func TestAgenticLoop_RetriesOnErrorEventBeforeVisibleOutput(t *testing.T) {
provider := &stubStreamingProvider{}
executor := tools.NewPulseToolExecutor(tools.ExecutorConfig{})
loop := NewAgenticLoop(provider, executor, "prompt")
callCount := 0
provider.chatStream = func(ctx context.Context, req providers.ChatRequest, callback providers.StreamCallback) error {
callCount++
if callCount == 1 {
callback(providers.StreamEvent{Type: "error", Data: providers.ErrorEvent{Message: "connection reset by peer"}})
return nil
}
callback(providers.StreamEvent{Type: "content", Data: providers.ContentEvent{Text: "recovered"}})
callback(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{}})
return nil
}
results, err := loop.Execute(context.Background(), "retry-error-event", []Message{{Role: "user", Content: "hi"}}, func(event StreamEvent) {})
if err != nil {
t.Fatalf("expected recovery after transient error event, got: %v", err)
}
if callCount != 2 {
t.Fatalf("expected 2 provider attempts, got %d", callCount)
}
if len(results) != 1 || results[0].Content != "recovered" {
t.Fatalf("unexpected results: %+v", results)
}
}
func TestAgenticLoop_DoesNotRetryErrorEventAfterVisibleOutput(t *testing.T) {
provider := &stubStreamingProvider{}
executor := tools.NewPulseToolExecutor(tools.ExecutorConfig{})
loop := NewAgenticLoop(provider, executor, "prompt")
callCount := 0
provider.chatStream = func(ctx context.Context, req providers.ChatRequest, callback providers.StreamCallback) error {
callCount++
callback(providers.StreamEvent{Type: "content", Data: providers.ContentEvent{Text: "partial"}})
callback(providers.StreamEvent{Type: "error", Data: providers.ErrorEvent{Message: "connection reset by peer"}})
return nil
}
_, err := loop.Execute(context.Background(), "no-retry-error-after-content", []Message{{Role: "user", Content: "hi"}}, func(event StreamEvent) {})
if err == nil {
t.Fatalf("expected error when stream emits error after visible output")
}
if callCount != 1 {
t.Fatalf("expected no retry after visible output, got %d attempts", callCount)
}
}
func TestCommandExtraction(t *testing.T) {
if cmd := getCommandFromInput(map[string]interface{}{"command": "ls"}); cmd != "ls" {
t.Fatalf("expected command to be extracted")
}
if cmd := getCommandFromInput(map[string]interface{}{}); cmd != "<unknown>" {
t.Fatalf("expected fallback command string")
}
}
func TestAgenticLoop_DoesNotAutoRecoverStructuredToolCall(t *testing.T) {
executor := tools.NewPulseToolExecutor(tools.ExecutorConfig{})
failCalls := 0
recoveryCalls := 0
executor.RegisterTool(tools.RegisteredTool{
Invocation: tools.StaticInvocation(agentcapabilities.ToolCallKindRead, agentcapabilities.MutationNone),
Definition: tools.Tool{
Name: "fail_tool",
InputSchema: tools.InputSchema{
Type: "object",
Properties: map[string]tools.PropertySchema{},
},
},
Handler: func(ctx context.Context, exec *tools.PulseToolExecutor, args map[string]interface{}) (tools.CallToolResult, error) {
failCalls++
return tools.NewToolResponseResult(tools.NewToolBlockedError(
tools.ErrCodeActionNotAllowed,
"blocked",
map[string]interface{}{
"policy_boundary": "resource requires model-selected follow-up context",
},
)), nil
},
})
executor.RegisterTool(tools.RegisteredTool{
Invocation: tools.StaticInvocation(agentcapabilities.ToolCallKindRead, agentcapabilities.MutationNone),
Definition: tools.Tool{
Name: "recovery_tool",
InputSchema: tools.InputSchema{
Type: "object",
Properties: map[string]tools.PropertySchema{
"value": {Type: "string"},
},
},
},
Handler: func(ctx context.Context, exec *tools.PulseToolExecutor, args map[string]interface{}) (tools.CallToolResult, error) {
recoveryCalls++
value, _ := args["value"].(string)
return tools.NewTextResult(value), nil
},
})
provider := &stubStreamingProvider{}
loop := NewAgenticLoop(provider, executor, "prompt")
callCount := 0
provider.chatStream = func(ctx context.Context, req providers.ChatRequest, callback providers.StreamCallback) error {
callCount++
switch callCount {
case 1:
callback(providers.StreamEvent{
Type: "tool_start",
Data: providers.ToolStartEvent{ID: "call_1", Name: "fail_tool"},
})
callback(providers.StreamEvent{
Type: "done",
Data: providers.DoneEvent{
ToolCalls: []providers.ToolCall{{
ID: "call_1",
Name: "fail_tool",
Input: map[string]interface{}{},
}},
},
})
case 2:
if len(req.Messages) != 3 {
t.Fatalf("expected assistant tool call plus tool result, got %d messages", len(req.Messages))
}
if req.Messages[2].ToolResult == nil || !req.Messages[2].ToolResult.IsError {
t.Fatalf("expected original blocked tool result, got %+v", req.Messages[2].ToolResult)
}
if strings.Contains(req.Messages[2].ToolResult.Content, "recovered through query") {
t.Fatalf("blocked result should not be replaced by auto-recovery output: %+v", req.Messages[2].ToolResult)
}
callback(providers.StreamEvent{
Type: "content",
Data: providers.ContentEvent{Text: "The tool was blocked; I need to decide the next step."},
})
callback(providers.StreamEvent{
Type: "done",
Data: providers.DoneEvent{},
})
default:
t.Fatalf("unexpected provider call %d", callCount)
}
return nil
}
results, err := loop.Execute(context.Background(), "structured-model-owned-recovery", []Message{{Role: "user", Content: "help"}}, func(event StreamEvent) {})
if err != nil {
t.Fatalf("expected model-owned recovery turn to continue, got %v", err)
}
if failCalls != 1 || recoveryCalls != 0 {
t.Fatalf("expected only the model-selected failing call to execute, got fail=%d recovery=%d", failCalls, recoveryCalls)
}
if len(results) != 3 {
t.Fatalf("expected assistant tool call, blocked tool result, and final response, got %+v", results)
}
if results[1].ToolResult == nil || !results[1].ToolResult.IsError {
t.Fatalf("expected blocked tool result in transcript, got %+v", results[1])
}
if results[2].Content != "The tool was blocked; I need to decide the next step." {
t.Fatalf("unexpected final response: %+v", results[2])
}
}
func TestAgenticLoop_NormalizesProviderToolCallsThroughSharedProjection(t *testing.T) {
executor := tools.NewPulseToolExecutor(tools.ExecutorConfig{})
var capturedArgs map[string]interface{}
executor.RegisterTool(tools.RegisteredTool{
Invocation: tools.StaticInvocation(agentcapabilities.ToolCallKindRead, agentcapabilities.MutationNone),
Definition: tools.Tool{
Name: "test_tool",
InputSchema: tools.InputSchema{
Type: "object",
Properties: map[string]tools.PropertySchema{
"value": {Type: "string"},
},
},
},
Handler: func(ctx context.Context, exec *tools.PulseToolExecutor, args map[string]interface{}) (tools.CallToolResult, error) {
capturedArgs = args
return tools.NewTextResult("normalized"), nil
},
})
provider := &stubStreamingProvider{}
loop := NewAgenticLoop(provider, executor, "prompt")
callCount := 0
sourceInput := map[string]interface{}{"value": "ok"}
provider.chatStream = func(ctx context.Context, req providers.ChatRequest, callback providers.StreamCallback) error {
callCount++
switch callCount {
case 1:
callback(providers.StreamEvent{
Type: "done",
Data: providers.DoneEvent{
ToolCalls: []providers.ToolCall{{
ID: "call-normalize",
Name: " test_tool ",
Input: sourceInput,
}},
},
})
case 2:
if len(req.Messages) != 3 {
t.Fatalf("expected user, normalized assistant call, and tool result, got %d messages", len(req.Messages))
}
call := req.Messages[1].ToolCalls[0]
if call.Name != "test_tool" {
t.Fatalf("provider continuation must receive normalized tool name, got %+v", call)
}
if req.Messages[2].ToolResult == nil || req.Messages[2].ToolResult.Content != "normalized" {
t.Fatalf("expected normalized tool result, got %+v", req.Messages[2].ToolResult)
}
callback(providers.StreamEvent{
Type: "content",
Data: providers.ContentEvent{Text: "Done."},
})
callback(providers.StreamEvent{
Type: "done",
Data: providers.DoneEvent{},
})
default:
t.Fatalf("unexpected provider call %d", callCount)
}
return nil
}
results, err := loop.Execute(context.Background(), "provider-call-normalize", []Message{{Role: "user", Content: "run it"}}, func(event StreamEvent) {})
if err != nil {
t.Fatalf("expected normalized provider tool call to execute, got %v", err)
}
if capturedArgs == nil || capturedArgs["value"] != "ok" {
t.Fatalf("expected normalized args to reach handler, got %#v", capturedArgs)
}
capturedArgs["value"] = "changed"
if sourceInput["value"] != "ok" {
t.Fatalf("handler args must not alias provider input: source=%#v captured=%#v", sourceInput, capturedArgs)
}
if len(results) != 3 {
t.Fatalf("expected assistant tool call, tool result, and final response, got %+v", results)
}
if len(results[0].ToolCalls) != 1 || results[0].ToolCalls[0].Name != "test_tool" {
t.Fatalf("session-visible assistant call must be normalized, got %+v", results[0].ToolCalls)
}
}
func TestAgenticLoop_CancelsUnavailableCurrentResourcePendingToolCall(t *testing.T) {
executor := tools.NewPulseToolExecutor(tools.ExecutorConfig{})
provider := &stubStreamingProvider{}
loop := NewAgenticLoop(provider, executor, "prompt")
var events []StreamEvent
callCount := 0
provider.chatStream = func(ctx context.Context, req providers.ChatRequest, callback providers.StreamCallback) error {
callCount++
switch callCount {
case 1:
callback(providers.StreamEvent{
Type: "tool_start",
Data: providers.ToolStartEvent{
ID: "call-read-dev",
Name: "pulse_read",
},
})
callback(providers.StreamEvent{
Type: "done",
Data: providers.DoneEvent{
ToolCalls: []providers.ToolCall{{
ID: "call-read-dev",
Name: "pulse_read",
Input: map[string]interface{}{
"action": "exec",
"target_host": "current_resource",
"command": "ls /dev | wc -l",
},
}},
},
})
case 2:
if len(req.Messages) != 3 {
t.Fatalf("expected user, assistant tool call, and hidden tool result, got %d messages", len(req.Messages))
}
toolResult := req.Messages[2].ToolResult
if toolResult == nil || !toolResult.IsError {
t.Fatalf("expected hidden current_resource tool result error, got %+v", toolResult)
}
if !strings.Contains(toolResult.Content, "CURRENT_RESOURCE_UNAVAILABLE") {
t.Fatalf("expected current resource block marker, got %q", toolResult.Content)
}
if !strings.Contains(toolResult.Content, "use pulse_query search to resolve its canonical ID") {
t.Fatalf("expected model-facing recovery instruction, got %q", toolResult.Content)
}
callback(providers.StreamEvent{
Type: "content",
Data: providers.ContentEvent{Text: "Which host, VM, container, app, or storage resource should I check?"},
})
callback(providers.StreamEvent{
Type: "done",
Data: providers.DoneEvent{},
})
default:
t.Fatalf("unexpected provider call %d", callCount)
}
return nil
}
results, err := loop.Execute(context.Background(), "current-resource-hidden-block", []Message{{Role: "user", Content: "how many devices in this"}}, func(event StreamEvent) {
events = append(events, event)
})
if err != nil {
t.Fatalf("expected hidden current_resource recovery turn, got %v", err)
}
if callCount != 2 {
t.Fatalf("expected recovery turn after hidden current_resource block, got %d provider calls", callCount)
}
toolStarts := 0
toolCancels := 0
for _, event := range events {
switch event.Type {
case "tool_start":
toolStarts++
case "tool_cancel":
toolCancels++
var data ToolCancelData
if err := json.Unmarshal(event.Data, &data); err != nil {
t.Fatalf("unmarshal tool_cancel event: %v", err)
}
if data.ID != "call-read-dev" || data.Name != "pulse_read" {
t.Fatalf("unexpected tool_cancel payload: %+v", data)
}
case "tool_end":
t.Fatalf("current_resource placeholder block should cancel the pending tool, not complete it: %+v", event)
}
}
if toolStarts != 1 || toolCancels != 1 {
t.Fatalf("expected one early tool_start and one tool_cancel, got starts=%d cancels=%d events=%+v", toolStarts, toolCancels, events)
}
if len(results) != 1 {
t.Fatalf("expected only final assistant message to be session-visible, got %d: %+v", len(results), results)
}
if len(results[0].ToolCalls) != 0 || results[0].ToolResult != nil {
t.Fatalf("expected hidden placeholder call to be absent from session-visible messages, got %+v", results[0])
}
if !strings.Contains(results[0].Content, "Which host") {
t.Fatalf("expected assistant to ask for target, got %q", results[0].Content)
}
}
// The model can interpret seed observations or explicitly report an access
// limit without a successful tool-call count certifying that interpretation.
func TestPatrolInvestigationCanConcludeFromSeedContext(t *testing.T) {
for _, seed := range []string{
"Current observation: the service endpoint is unreachable. No host agent is connected. The cause is unknown.",
"Collection is unavailable. No current service or host observation is supplied. The cause is unknown.",
} {
t.Run(seed, func(t *testing.T) {
const conclusion = "The cause is unknown. Host access is unavailable, and no action was performed."
calls := 0
provider := &stubStreamingProvider{}
provider.chatStream = func(_ context.Context, req providers.ChatRequest, callback providers.StreamCallback) error {
calls++
if req.ToolChoice != nil && req.ToolChoice.Type == providers.ToolChoiceRequired {
t.Fatal("a tool call was forced despite the supplied context")
}
if len(req.Messages) == 0 || req.Messages[0].Content != seed {
t.Fatalf("seed context changed: %+v", req.Messages)
}
callback(providers.StreamEvent{Type: "content", Data: providers.ContentEvent{Text: conclusion}})
callback(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{}})
return nil
}
executor := tools.NewPulseToolExecutor(tools.ExecutorConfig{})
executor.ApplyExecutionProfile(tools.ProfilePatrolInvestigation)
loop := NewAgenticLoop(provider, executor, "Investigate the supplied issue.")
loop.SetExecutionProfile(tools.ProfilePatrolInvestigation)
loop.SetMaxTurns(3)
result, err := loop.ExecuteWithTools(context.Background(), "seed-context", []Message{{Role: "user", Content: seed}}, []providers.Tool{{Name: agentcapabilities.PulseQueryToolName}}, func(StreamEvent) {})
if err != nil || calls != 1 || loop.GetTotalEvidenceCalls() != 0 {
t.Fatalf("unexpected forced investigation: calls=%d evidence=%d error=%v", calls, loop.GetTotalEvidenceCalls(), err)
}
if len(result) != 1 || result[0].Content != conclusion {
t.Fatalf("uncertainty was discarded: %+v", result)
}
})
}
}
func TestPatrolInvestigationPreservesFailedReadsAndUncertainty(t *testing.T) {
t.Setenv("PULSE_STRICT_RESOLUTION", "false")
for _, policyBlocked := range []bool{false, true} {
t.Run(fmt.Sprintf("policyBlocked_%v", policyBlocked), func(t *testing.T) {
const conclusion = "The requested evidence could not be collected. The cause remains unknown. No action was performed."
cfg := tools.ExecutorConfig{StateProvider: &mockStateProvider{}}
call := providers.ToolCall{ID: "unavailable-evidence", Name: agentcapabilities.PulseQueryToolName, Input: map[string]interface{}{"action": "unsupported"}}
if policyBlocked {
cfg.AgentServer = policyBlockedEvidenceAgentServer{}
call.Name = agentcapabilities.PulseReadToolName
call.Input = map[string]interface{}{"action": "exec", "command": "uptime", "target_host": "host-1"}
}
provider := &stubStreamingProvider{}
calls := 0
var observed string
provider.chatStream = func(_ context.Context, req providers.ChatRequest, callback providers.StreamCallback) error {
calls++
if calls == 1 {
callback(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{ToolCalls: []providers.ToolCall{call}}})
return nil
}
if calls != 2 {
t.Fatalf("unexpected forced follow-up turn %d", calls)
}
if providerToolIsAdvertised(req.Tools, call.Name) {
t.Fatal("exhausted evidence capability remained available")
}
for _, msg := range req.Messages {
if msg.ToolResult != nil && msg.ToolResult.ToolUseID == call.ID {
observed = msg.ToolResult.Content
if policyBlocked {
if !agentcapabilities.HasPolicyBlockedToolMarker(observed) {
t.Fatalf("policy boundary lost: %q", observed)
}
} else if !msg.ToolResult.IsError {
t.Fatal("failed read became successful evidence")
}
}
}
if observed == "" || strings.Contains(observed, "Patrol evidence checkpoint") || strings.Contains(observed, "Patrol evidence budget") {
t.Fatalf("result missing or altered by diagnostic policy: %q", observed)
}
callback(providers.StreamEvent{Type: "content", Data: providers.ContentEvent{Text: conclusion}})
callback(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{}})
return nil
}
executor := tools.NewPulseToolExecutor(cfg)
executor.ApplyExecutionProfile(tools.ProfilePatrolInvestigation)
loop := NewAgenticLoop(provider, executor, "Investigate the supplied issue.")
loop.SetExecutionProfile(tools.ProfilePatrolInvestigation)
loop.SetMaxEvidenceCalls(1)
loop.SetMaxTurns(3)
result, err := loop.ExecuteWithTools(context.Background(), "missing-access", []Message{{Role: "user", Content: "Investigate. No current host observation is available."}}, []providers.Tool{{Name: call.Name}, {Name: agentcapabilities.PatrolProposeActionToolName}}, func(StreamEvent) {})
if err != nil || calls != 2 || loop.GetTotalEvidenceCalls() != 1 {
t.Fatalf("failed read prevented an honest conclusion: calls=%d evidence=%d error=%v", calls, loop.GetTotalEvidenceCalls(), err)
}
if len(result) == 0 || result[len(result)-1].Content != conclusion {
t.Fatalf("uncertain conclusion lost: %+v", result)
}
found := false
for _, msg := range result {
if msg.ToolResult != nil && msg.ToolResult.ToolUseID == call.ID {
found = true
if msg.ToolResult.Content != observed {
t.Fatalf("persisted result differs from model evidence: %q != %q", msg.ToolResult.Content, observed)
}
}
}
if !found {
t.Fatal("failed read missing from retained transcript")
}
})
}
}
// A multi-resource investigation may need several tool-only turns. Call counts
// must neither alter observations nor stand in for the model's conclusion.
func TestAgenticLoopPreservesEvidenceThroughLongReadSequence(t *testing.T) {
for _, profile := range []tools.ExecutionProfile{tools.ProfileInteractiveAssistant, tools.ProfilePatrolDetection} {
t.Run(fmt.Sprint(profile), func(t *testing.T) {
const conclusion = "No matching resources were found. The requested diagnosis remains unknown."
provider := &stubStreamingProvider{}
calls := 0
observations := make(map[string]string)
provider.chatStream = func(_ context.Context, req providers.ChatRequest, callback providers.StreamCallback) error {
calls++
for _, msg := range req.Messages {
if msg.ToolResult == nil {
continue
}
result := msg.ToolResult
if result.IsError || strings.Contains(result.Content, "[System:") {
t.Fatalf("read result failed or contains injected policy: %+v", result)
}
if prior, found := observations[result.ToolUseID]; found && prior != result.Content {
t.Fatalf("observation changed between provider turns: %q != %q", prior, result.Content)
}
observations[result.ToolUseID] = result.Content
}
if !providerToolIsAdvertised(req.Tools, agentcapabilities.PulseQueryToolName) {
t.Fatalf("read capability withdrawn before configured run limit on turn %d", calls)
}
if calls <= 5 {
var batch []providers.ToolCall
for resource := 0; resource < 4; resource++ {
id := fmt.Sprintf("resource-%d-%d", calls, resource)
batch = append(batch, providers.ToolCall{ID: id, Name: agentcapabilities.PulseQueryToolName, Input: map[string]interface{}{"action": "search", "query": id}})
}
callback(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{ToolCalls: batch}})
} else {
callback(providers.StreamEvent{Type: "content", Data: providers.ContentEvent{Text: conclusion}})
callback(providers.StreamEvent{Type: "done", Data: providers.DoneEvent{}})
}
return nil
}
executor := tools.NewPulseToolExecutor(tools.ExecutorConfig{StateProvider: &mockStateProvider{}})
executor.ApplyExecutionProfile(profile)
loop := NewAgenticLoop(provider, executor, "Investigate the requested resources.")
loop.SetExecutionProfile(profile)
loop.SetMaxTurns(8)
result, err := loop.ExecuteWithTools(context.Background(), "long-read-sequence", []Message{{Role: "user", Content: "Check these twenty resources."}}, []providers.Tool{{Name: agentcapabilities.PulseQueryToolName}}, func(StreamEvent) {})
if err != nil || calls != 6 || len(observations) != 20 {
t.Fatalf("read sequence was curtailed: calls=%d observations=%d error=%v", calls, len(observations), err)
}
if len(result) == 0 || result[len(result)-1].Content != conclusion {
t.Fatalf("model conclusion lost: %+v", result)
}
for _, msg := range result {
if msg.ToolResult != nil && observations[msg.ToolResult.ToolUseID] != msg.ToolResult.Content {
t.Fatalf("durable evidence differs from provider observation: %+v", msg.ToolResult)
}
}
})
}
}