An open-source, privacy-first, self-hosted knowledge workspace where humans and AI agents work together 开源、隐私优先、自托管的知识工作空间,让人与智能体在此协作
TypeScript agentic-aiai-agentknowledge-baselocal-firstmarkdownmcpmindmapnote-takingself-hostedsiyuanworkspace
// SiYuan - https://github.com/siyuan-note/siyuan
// Copyright (c) 2020-present, b3log.org
// SPDX-License-Identifier: AGPL-3.0-or-later
package agent
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/siyuan-note/siyuan/kernel/mcp/tools"
kernelModel "github.com/siyuan-note/siyuan/kernel/model"
"github.com/siyuan-note/siyuan/kernel/util"
)
const (
testToolRoundLimitMessage = "Tool call round limit reached. Remaining tool calls were not executed; send another message to continue"
testToolRoundLimitToolName = "test_round_limit"
)
func TestAgentChatToolRoundLimit(t *testing.T) {
for _, protocol := range []string{"openai", util.OpenAIProtocolResponses, util.AnthropicProtocolMessages} {
for _, tc := range []struct {
name string
chunks []string
structured bool
rejected bool
budget int
reasoning string
}{
{name: "summary", chunks: []string{"Completed the read. ", "The edit remains unfinished."}, budget: 1},
{name: "DSML", chunks: []string{"Preparing an edit <", "||DS", "ML|| calls><invoke name=\"file\">edit</invoke>"}, rejected: true, budget: 1},
{name: "tool calls token", chunks: []string{"<|tool_", "calls|>file edit"}, rejected: true, budget: 1},
{name: "Anthropic markup", chunks: []string{"<antml:", "invoke name=\"file\">edit</antml:invoke>"}, rejected: true, budget: 1},
{name: "structured call", structured: true, rejected: true, budget: 1},
{name: "unlimited", chunks: []string{"Task completed."}, budget: 0},
{name: "below limit", chunks: []string{"Task completed."}, budget: 2},
{name: "reasoning markup", chunks: []string{"An edit was requested."}, reasoning: "<|tool_calls|>file edit", rejected: true, budget: 1},
} {
if tc.reasoning != "" && protocol != "openai" {
continue
}
t.Run(protocol+"/"+tc.name, func(t *testing.T) {
executed := setupToolRoundLimitAgentTest(t, tc.budget)
limited := tc.budget == 1
var requests atomic.Int32
var followup atomic.Bool
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var payload map[string]json.RawMessage
if err := json.NewDecoder(r.Body).Decode(&payload); err != nil {
t.Error(err)
return
}
attempt := requests.Add(1)
if attempt > 2 && (!followup.Load() || attempt != 3) {
t.Error("model requested again after the final summary")
w.WriteHeader(http.StatusBadRequest)
return
}
encoded, _ := json.Marshal(payload)
if attempt == 1 || attempt == 3 || !limited {
if len(payload["tools"]) == 0 || strings.Contains(string(encoded), "Tool call round limit reached") {
t.Error("tools were limited before the budget was spent")
}
} else {
if raw := string(payload["tools"]); raw != "" && raw != "[]" && raw != "null" {
t.Errorf("tools remained available at the limit: %s", raw)
}
if !strings.Contains(string(encoded), "Tool call round limit reached") || !strings.Contains(string(encoded), "completed read") {
t.Errorf("summary request lost the limit instruction or completed tool result: %s", encoded)
}
}
tool := ""
if attempt == 1 || (attempt == 2 && tc.structured) {
tool = testToolRoundLimitToolName
}
chunks := tc.chunks
if attempt == 3 {
chunks = []string{"Continued normally."}
}
content := strings.Join(chunks, "")
switch protocol {
case util.AnthropicProtocolMessages:
writeAnthropicAgentResponse(t, w, "signature", content, tool, true)
case util.OpenAIProtocolResponses:
flusher := prepareTestStream(t, w)
var output []any
if tool != "" {
output = []any{map[string]any{"type": "function_call", "id": "fc-1", "call_id": "call-1", "name": tool, "arguments": `{"action":"list"}`, "status": "completed"}}
} else {
output = []any{map[string]any{"type": "message", "id": "msg-1", "role": "assistant", "status": "completed", "content": []any{map[string]any{"type": "output_text", "text": content}}}}
}
writeResponsesTestCompleted(t, w, flusher, "resp-1", output, 10, 5)
default:
flusher := prepareTestStream(t, w)
if tool != "" {
_, _ = fmt.Fprintf(w, "data: {\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"call-1\",\"type\":\"function\",\"function\":{\"name\":%q,\"arguments\":\"{\\\"action\\\":\\\"list\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]}\n\n", tool)
flusher.Flush()
} else {
if attempt == 2 && tc.reasoning != "" {
_, _ = fmt.Fprintf(w, "data: {\"choices\":[{\"index\":0,\"delta\":{\"reasoning_content\":%q}}]}\n\n", tc.reasoning)
flusher.Flush()
}
for _, chunk := range chunks {
writeTestStreamChunk(t, w, flusher, chunk)
}
}
writeTestStreamDone(t, w, flusher)
}
}))
defer server.Close()
var content strings.Builder
errorSeen, usageSeen, doneSeen := false, false, false
for event := range AgentChat(context.Background(), util.NewAIClientWithModel("key", server.URL+"/v1", "test-model"), protocol, "test-model", "", 0,
testSessionID, "user-1", 1, "read and edit", nil, "English", nil, EditorContext{}, nil, false,
time.Second, 0, "", time.Second, time.Second) {
switch event.Type {
case "content":
content.WriteString(event.Token)
case "reasoning":
if tc.reasoning != "" && event.Token == tc.reasoning {
t.Error("raw tool markup was exposed as reasoning")
}
case "done":
doneSeen = true
case "error":
errorSeen = true
if event.Error != testToolRoundLimitMessage {
t.Errorf("unexpected interruption: %q", event.Error)
}
case "usage":
usageSeen = true
case "permission":
if event.PermissionMode == "allowSession" {
t.Error("round limit changed write permissions")
}
}
}
wantContent, wantDelta := strings.Join(tc.chunks, ""), 2
if tc.rejected {
wantContent, wantDelta = "", 1
}
if content.String() != wantContent || errorSeen != limited || doneSeen == limited || !usageSeen || requests.Load() != 2 || executed.Load() != 1 {
t.Fatalf("unexpected final response: content=%q error=%v done=%v usage=%v requests=%d executed=%d", content.String(), errorSeen, doneSeen, usageSeen, requests.Load(), executed.Load())
}
runtime, err := loadRuntimeState(testSessionID)
if err != nil {
t.Fatal(err)
}
turn := runtime.ActiveTurn
wantState := "finished"
if limited {
wantState = "interrupted"
}
if turn == nil || turn.State != wantState || len(turn.Delta) != wantDelta || turn.DraftContent != "" {
t.Fatalf("unexpected saved turn: %#v", turn)
}
if len(turn.Delta[0].ToolCalls) != 1 || turn.Delta[0].ToolCalls[0].State != "finished" || !strings.Contains(turn.Delta[0].ToolCalls[0].Result, "completed read") {
t.Fatalf("completed tool result was lost: %#v", turn.Delta[0])
}
if kernelModel.Conf.AI.Load().Agent.MaxToolCallRounds != tc.budget {
t.Error("turn changed the global tool budget")
}
// 提交中断轮次后,下一条用户消息恢复工具预算,且不继承临时上限指令。
recovered, err := GetSession(testSessionID)
if err != nil {
t.Fatal(err)
}
recovered["expectedRevision"], recovered["commitTurnID"] = int64(1), turn.TurnID
if _, _, err = SaveSessionState(marshalSession(t, recovered)); err != nil {
t.Fatal(err)
}
recovered, err = GetSession(testSessionID)
if err != nil {
t.Fatal(err)
}
recovered["entries"] = append(recovered["entries"].([]any), map[string]any{"id": "user-2", "type": "user", "content": "continue"})
recovered["expectedRevision"] = int64(2)
if _, _, err = SaveSessionState(marshalSession(t, recovered)); err != nil {
t.Fatal(err)
}
followup.Store(true)
doneSeen = false
for event := range AgentChat(context.Background(), util.NewAIClientWithModel("key", server.URL+"/v1", "test-model"), protocol, "test-model", "", 0,
testSessionID, "user-2", 3, "continue", nil, "English", nil, EditorContext{}, nil, false,
time.Second, 0, "", time.Second, time.Second) {
if event.Type == "error" {
t.Fatalf("follow-up failed: %s", event.Error)
}
if event.Type == "done" {
doneSeen = true
}
}
if !doneSeen || requests.Load() != 3 {
t.Fatalf("follow-up did not complete normally: done=%v requests=%d", doneSeen, requests.Load())
}
})
}
}
}
func TestAgentChatToolRoundLimitDiscardsUnvalidatedStream(t *testing.T) {
for _, cancelStream := range []bool{false, true} {
t.Run(fmt.Sprintf("cancel=%v", cancelStream), func(t *testing.T) {
executed := setupToolRoundLimitAgentTest(t, 1)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
var requests atomic.Int32
streamStarted := make(chan struct{})
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
flusher := prepareTestStream(t, w)
if requests.Add(1) == 1 {
_, _ = fmt.Fprintf(w, "data: {\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"call-1\",\"type\":\"function\",\"function\":{\"name\":%q,\"arguments\":\"{\\\"action\\\":\\\"list\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]}\n\n", testToolRoundLimitToolName)
flusher.Flush()
writeTestStreamDone(t, w, flusher)
return
}
writeTestStreamChunk(t, w, flusher, "<|tool_calls|>file edit")
close(streamStarted)
<-r.Context().Done()
}))
defer server.Close()
idleTimeout := 50 * time.Millisecond
if cancelStream {
idleTimeout = time.Second
}
events := AgentChat(ctx, newTestOpenAIClient(server.URL), "openai", "test-model", "", 0,
testSessionID, "user-1", 1, "read and edit", nil, "English", nil, EditorContext{}, nil, false,
time.Second, 0, "", time.Second, idleTimeout)
if cancelStream {
select {
case <-streamStarted:
cancel()
case <-time.After(5 * time.Second):
t.Fatal("summary stream did not start")
}
}
errorSeen := false
for event := range events {
if event.Type == "content" || event.Type == "reasoning" || event.Type == "done" {
t.Errorf("unvalidated response was exposed: %#v", event)
}
if event.Type == "error" {
errorSeen = true
}
}
if !cancelStream && !errorSeen {
t.Error("stream failure was not reported")
}
runtime, err := loadRuntimeState(testSessionID)
if err != nil {
t.Fatal(err)
}
turn := runtime.ActiveTurn
if turn == nil || turn.State != "interrupted" || len(turn.Delta) != 1 || turn.DraftContent != "" || turn.DraftRoundID != "" {
t.Fatalf("unvalidated response was checkpointed: %#v", turn)
}
if executed.Load() != 1 || requests.Load() != 2 {
t.Fatalf("stream failure retried work: executed=%d requests=%d", executed.Load(), requests.Load())
}
})
}
}
func setupToolRoundLimitAgentTest(t *testing.T, budget int) *atomic.Int32 {
t.Helper()
setupCompactionAgentTest(t)
useTestToolRoundLimitLanguage(t)
kernelModel.Conf.AI.Load().Agent.MaxToolCallRounds = budget
var executed atomic.Int32
tools.SetTool(testToolRoundLimitToolName, &tools.Tool{
Name: testToolRoundLimitToolName, Source: "native", ReadOnlyHint: true,
InputSchema: tools.ToolSchema{Type: "object", Properties: map[string]tools.Property{"action": {Type: "string"}}},
ActionEffects: map[string]tools.ToolEffects{"list": {LocalRead: true}},
Handler: func(map[string]any) (tools.CallToolResult, error) {
executed.Add(1)
return tools.CallToolResult{Content: []tools.ContentItem{{Type: "text", Text: "completed read"}}}, nil
},
})
t.Cleanup(func() { tools.RemoveTool(testToolRoundLimitToolName) })
if _, _, err := SaveSessionState(marshalSession(t, map[string]any{
"id": testSessionID, "title": "round limit", "createdAt": 1, "updatedAt": 1,
"entries": []any{map[string]any{"id": "user-1", "type": "user", "content": "read and edit"}},
})); err != nil {
t.Fatal(err)
}
return &executed
}
func useTestToolRoundLimitLanguage(t *testing.T) {
t.Helper()
originalLangs := util.Langs
util.Langs = map[string]map[int]string{"en": {440: testToolRoundLimitMessage}}
t.Cleanup(func() { util.Langs = originalLangs })
}