Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(agent): preserve multimodal content during message sanitization (#3949)
澍2f3b04c52e00
diff --git a/internal/agent/compaction/fileops.go b/internal/agent/compaction/fileops.goindex 77060c571..295aaf75b 100644--- a/internal/agent/compaction/fileops.go+++ b/internal/agent/compaction/fileops.go@@ -63,7 +63,7 @@ func extractFileOps(previousSummary string, groups ...[]chat.Message) fileOps {ops.inherit(previousSummary)for _, group := range groups {for _, msg := range group {- ops.inherit(msg.Content)+ ops.inherit(messageText(msg))for _, tc := range msg.ToolCalls {path := toolCallPath(tc.Function.Arguments)if path == "" {diff --git a/internal/agent/compaction/multimodal_overflow_test.go b/internal/agent/compaction/multimodal_overflow_test.gonew file mode 100644index 000000000..b5d097cdd--- /dev/null+++ b/internal/agent/compaction/multimodal_overflow_test.go@@ -0,0 +1,62 @@+package compaction++import (+ "strings"+ "testing"++ agenttools "github.com/Tencent/WeKnora/internal/agent/tools"+ "github.com/Tencent/WeKnora/internal/models/chat"+ "github.com/stretchr/testify/require"+)++func TestOverflowAfterSanitizationPreservesPreviousSummary(t *testing.T) {+ const marker = "KEEP_PRIOR_USER_DECISION_7E93"+ live := reactTurn(40)+ summary := SummaryMessage(marker + "\n<modified-files>\n/workspace/prior.txt\n</modified-files>")+ image := chat.Message{Role: "user", MultiContent: []chat.MessageContentPart{+ {Type: "image_url", ImageURL: &chat.ImageURL{URL: "data:image/png;base64,AA=="}},+ {Type: "text", Text: "continue from this screenshot"},+ }}+ messages := append([]chat.Message{live[0], summary, image}, live[2:]...)+ sanitized := agenttools.SanitizeMessages(messages)+ require.Equal(t, chat.MessageKindCompactionSummary, sanitized[1].Kind)+ require.Empty(t, sanitized[1].Content, "exercise the multipart-only summary produced by sanitization")+ require.NotEmpty(t, sanitized[1].MultiContent)++ llm := &stubChat{response: "## Goal\ncontinue the task"}+ c := New(llm, newEstimator(t), testSettings())+ result, err := c.Compact(t.Context(), sanitized, ReasonOverflow)+ require.NoError(t, err)+ require.NotEmpty(t, llm.prompts)+ require.Contains(t, strings.Join(llm.prompts, "\n"), marker)+ require.Contains(t, strings.Join(llm.prompts, "\n"), "<previous-summary>")+ require.Contains(t, result.Summary, "/workspace/prior.txt")+ require.Equal(t, marker+"\n<modified-files>\n/workspace/prior.txt\n</modified-files>",+ unwrapSummary(messages[1].Content), "caller-owned messages must not be changed")+}++func TestCompactionSerializesMultipartTextWithoutLegacyDuplication(t *testing.T) {+ for _, role := range []string{"user", "assistant", "tool"} {+ t.Run(role, func(t *testing.T) {+ message := chat.Message{Role: role, Content: "stale legacy text", MultiContent: []chat.MessageContentPart{+ {Type: "text", Text: "keep first"},+ {Type: "image_url", ImageURL: &chat.ImageURL{URL: "https://example.com/image.png"}},+ {Type: "text", Text: " and second"},+ }}+ for _, rendered := range []string{serializeConversation([]chat.Message{message}), archiveLine(&message)} {+ require.Contains(t, rendered, "keep first and second")+ require.NotContains(t, rendered, "stale legacy text")+ require.NotContains(t, rendered, "https://example.com")+ }+ })+ }+}++func TestCompactionInheritsMultipartFileOperations(t *testing.T) {+ message := chat.Message{Role: "user", MultiContent: []chat.MessageContentPart{+ {Type: "text", Text: "<modified-files>\n/workspace/kept.txt\n</modified-files>"},+ }}+ ops := extractFileOps("", []chat.Message{message})+ _, modified := ops.resolve()+ require.Equal(t, []string{"/workspace/kept.txt"}, modified)+}diff --git a/internal/agent/compaction/prepare.go b/internal/agent/compaction/prepare.goindex 18d529ad6..299e4d4a7 100644--- a/internal/agent/compaction/prepare.go+++ b/internal/agent/compaction/prepare.go@@ -66,7 +66,7 @@ func Prepare(messages []chat.Message, s Settings, estimator *agenttoken.Estimatoif messages[i].Kind != chat.MessageKindCompactionSummary {continue}- previousSummary = unwrapSummary(messages[i].Content)+ previousSummary = unwrapSummary(messageText(messages[i]))boundaryStart = i + 1break}diff --git a/internal/agent/compaction/serialize.go b/internal/agent/compaction/serialize.goindex f0a6bcab7..2821b911a 100644--- a/internal/agent/compaction/serialize.go+++ b/internal/agent/compaction/serialize.go@@ -31,11 +31,12 @@ func serializeConversation(messages []chat.Message) string {var parts []stringfor i := range messages {msg := &messages[i]+ text := messageText(*msg)switch msg.Role {case "system":continuecase "user":- if content := truncate(msg.Content, textMaxChars); content != "" {+ if content := truncate(text, textMaxChars); content != "" {parts = append(parts, "[User]: "+content)}case "assistant":@@ -43,14 +44,14 @@ func serializeConversation(messages []chat.Message) string {parts = append(parts,"[Assistant thinking]: "+truncate(msg.ReasoningContent, textMaxChars))}- if msg.Content != "" {- parts = append(parts, "[Assistant]: "+truncate(msg.Content, textMaxChars))+ if text != "" {+ parts = append(parts, "[Assistant]: "+truncate(text, textMaxChars))}if calls := serializeToolCalls(msg.ToolCalls); calls != "" {parts = append(parts, "[Assistant tool calls]: "+calls)}case "tool":- if content := truncate(msg.Content, toolResultMaxChars); content != "" {+ if content := truncate(text, toolResultMaxChars); content != "" {parts = append(parts, fmt.Sprintf("[Tool result %s]: %s", msg.Name, content))}}@@ -143,16 +144,33 @@ func rawArchive(messages []chat.Message, budget int, estimator *agenttoken.Estim}func archiveLine(msg *chat.Message) string {+ text := messageText(*msg)switch msg.Role {case "user":- return fmt.Sprintf("- User: %s\n", truncate(msg.Content, 500))+ return fmt.Sprintf("- User: %s\n", truncate(text, 500))case "assistant":if calls := serializeToolCalls(msg.ToolCalls); calls != "" {- return fmt.Sprintf("- Assistant [%s]: %s\n", calls, truncate(msg.Content, 500))+ return fmt.Sprintf("- Assistant [%s]: %s\n", calls, truncate(text, 500))}- return fmt.Sprintf("- Assistant: %s\n", truncate(msg.Content, 500))+ return fmt.Sprintf("- Assistant: %s\n", truncate(text, 500))case "tool":- return fmt.Sprintf("- Tool[%s]: %s\n", msg.Name, truncate(msg.Content, 500))+ return fmt.Sprintf("- Tool[%s]: %s\n", msg.Name, truncate(text, 500))}return ""}++// messageText follows the providers' preference for multipart content. The+// sanitizer can move a compaction summary's text into MultiContent while+// retaining its Kind, so scalar Content alone is not a complete transcript.+func messageText(msg chat.Message) string {+ if len(msg.MultiContent) == 0 {+ return msg.Content+ }+ var text strings.Builder+ for _, part := range msg.MultiContent {+ if part.Type == "text" {+ text.WriteString(part.Text)+ }+ }+ return text.String()+}diff --git a/internal/agent/resume_test.go b/internal/agent/resume_test.goindex 8f4b6aa3c..0ccfaeff3 100644--- a/internal/agent/resume_test.go+++ b/internal/agent/resume_test.go@@ -152,10 +152,24 @@ func TestResumeReplaysSteeringAtItsOriginalBoundaries(t *testing.T) {}}}}_, err = f.engine.Resume(t.Context(), "sess-1", "msg-1", nil, data, answeredPrompt("call-b"))require.NoError(t, err)- var contents []string+ var contents, images []stringfor _, m := range model.calls[0] {- contents = append(contents, m.Content)+ if len(m.MultiContent) == 0 {+ contents = append(contents, m.Content)+ images = append(images, m.Images...)+ continue+ }+ for _, part := range m.MultiContent {+ switch part.Type {+ case "text":+ contents = append(contents, part.Text)+ case "image_url":+ require.NotNil(t, part.ImageURL)+ images = append(images, part.ImageURL.URL)+ }+ }}+ require.Equal(t, []string{"https://img/1.png"}, images)// The provider normalization can merge adjacent user messages, including// the original query and the first steering message.transcript := strings.Join(contents, "\n")diff --git a/internal/agent/tools/sanitize_messages.go b/internal/agent/tools/sanitize_messages.goindex f58892f96..3188c57dc 100644--- a/internal/agent/tools/sanitize_messages.go+++ b/internal/agent/tools/sanitize_messages.go@@ -49,8 +49,16 @@ func SanitizeMessages(messages []chat.Message) []chat.Message {// parts go along for the same reason as tool calls: dropping// them leaves the model reading captions of images it never got.merged.ToolCalls = concatCopy(prev.ToolCalls, msg.ToolCalls)- merged.Images = concatCopy(prev.Images, msg.Images)- merged.MultiContent = concatCopy(prev.MultiContent, msg.MultiContent)+ if len(prev.Images)+len(msg.Images)+len(prev.MultiContent)+len(msg.MultiContent) > 0 {+ // Preserve each message's image/text order in one representation.+ // Providers prefer MultiContent over Content; retaining only the+ // latter would lose the appended text and images on the wire.+ parts := appendSanitizedContent(nil, prev)+ parts = append(parts, chat.MessageContentPart{Type: "text", Text: "\n\n"})+ merged.MultiContent = appendSanitizedContent(parts, msg)+ merged.Content = ""+ merged.Images = nil+ }result[len(result)-1] = mergedcontinue}@@ -82,6 +90,24 @@ func concatCopy[T any](a, b []T) []T {return append(append(out, a...), b...)}+// appendSanitizedContent follows the providers' preference for MultiContent,+// expanding the legacy Content/Images representation only when needed.+// The destination is separate from the input messages' backing arrays.+func appendSanitizedContent(parts []chat.MessageContentPart, msg chat.Message) []chat.MessageContentPart {+ if len(msg.MultiContent) > 0 {+ return append(parts, msg.MultiContent...)+ }+ for _, url := range msg.Images {+ parts = append(parts, chat.MessageContentPart{+ Type: "image_url", ImageURL: &chat.ImageURL{URL: url, Detail: "auto"},+ })+ }+ if msg.Content != "" {+ parts = append(parts, chat.MessageContentPart{Type: "text", Text: msg.Content})+ }+ return parts+}+// hasMatchingToolCall checks if any preceding assistant message has a tool call with the given ID.func hasMatchingToolCall(messages []chat.Message, toolCallID string) bool {for i := len(messages) - 1; i >= 0; i-- {diff --git a/internal/agent/tools/sanitize_messages_multimodal_test.go b/internal/agent/tools/sanitize_messages_multimodal_test.gonew file mode 100644index 000000000..a45373b92--- /dev/null+++ b/internal/agent/tools/sanitize_messages_multimodal_test.go@@ -0,0 +1,133 @@+package tools++import (+ "encoding/json"+ "testing"++ "github.com/Tencent/WeKnora/internal/models/api"+ "github.com/Tencent/WeKnora/internal/models/api/openaicompletions"+ "github.com/Tencent/WeKnora/internal/models/chat"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++func TestSanitizeMessagesPreservesMultimodalContent(t *testing.T) {+ image := chat.MessageContentPart{+ Type: "image_url", ImageURL: &chat.ImageURL{URL: "https://example.com/a.png", Detail: "high"},+ }+ text := chat.MessageContentPart{Type: "text", Text: "describe this"}+ for i, msg := range []chat.Message{+ {Role: "user", Images: []string{"https://example.com/a.png"}},+ {Role: "user", MultiContent: []chat.MessageContentPart{image}},+ {Role: "user", MultiContent: []chat.MessageContentPart{text, image}},+ } {+ t.Run([]string{"legacy image", "image part", "text and image parts"}[i], func(t *testing.T) {+ got := SanitizeMessages([]chat.Message{msg})+ require.Equal(t, []chat.Message{msg}, got, "nonempty multimodal content must not be discarded")+ })+ }+}++func TestSanitizeMessagesKeepsImagesInProviderRequest(t *testing.T) {+ client := openaicompletions.New(openaicompletions.Config{+ Endpoint: api.Endpoint{BaseURL: "https://example.com/v1", Model: "vision-model"},+ Settings: api.DefaultOpenAICompletions(),+ })+ for _, stream := range []bool{false, true} {+ body, err := client.BuildRequestBody(SanitizeMessages([]chat.Message{+ {Role: "user", Content: "earlier conversation summary"},+ {Role: "user", MultiContent: []chat.MessageContentPart{+ {Type: "text", Text: "compare the screenshots"},+ {Type: "image_url", ImageURL: &chat.ImageURL{URL: "https://example.com/first.png", Detail: "high"}},+ }},+ {Role: "user", Content: "second screenshot", Images: []string{"https://example.com/second.png"}},+ }), nil, stream)+ require.NoError(t, err)+ data, err := json.Marshal(body)+ require.NoError(t, err)+ var request struct {+ Messages []struct {+ Role string `json:"role"`+ Content json.RawMessage `json:"content"`+ } `json:"messages"`+ }+ require.NoError(t, json.Unmarshal(data, &request))+ require.Len(t, request.Messages, 1)+ assert.Equal(t, "user", request.Messages[0].Role)+ assert.JSONEq(t, `[+ {"type":"text","text":"earlier conversation summary"},+ {"type":"text","text":"\n\n"},+ {"type":"text","text":"compare the screenshots"},+ {"type":"image_url","image_url":{"url":"https://example.com/first.png","detail":"high"}},+ {"type":"text","text":"\n\n"},+ {"type":"image_url","image_url":{"url":"https://example.com/second.png","detail":"auto"}},+ {"type":"text","text":"second screenshot"}+ ]`, string(request.Messages[0].Content))+ }+}++func TestSanitizeMessagesMergesMultimodalContentInOrder(t *testing.T) {+ imageA := chat.MessageContentPart{+ Type: "image_url", ImageURL: &chat.ImageURL{URL: "https://example.com/a.png", Detail: "auto"},+ }+ imageB := chat.MessageContentPart{+ Type: "image_url", ImageURL: &chat.ImageURL{URL: "https://example.com/b.png", Detail: "high"},+ }+ legacyB := chat.MessageContentPart{+ Type: "image_url", ImageURL: &chat.ImageURL{URL: imageB.ImageURL.URL, Detail: "auto"},+ }+ textA := chat.MessageContentPart{Type: "text", Text: "first image"}+ textB := chat.MessageContentPart{Type: "text", Text: "second image"}+ separator := chat.MessageContentPart{Type: "text", Text: "\n\n"}+ cases := []struct {+ name string+ messages []chat.Message+ want []chat.MessageContentPart+ }{+ {"tool image messages", []chat.Message{+ {Role: "user", Content: textA.Text, Images: []string{imageA.ImageURL.URL}},+ {Role: "user", Content: textB.Text, Images: []string{imageB.ImageURL.URL}},+ }, []chat.MessageContentPart{imageA, textA, separator, legacyB, textB}},+ {"summary before multimodal user", []chat.Message{+ {Role: "user", Content: "compaction summary"},+ {Role: "user", MultiContent: []chat.MessageContentPart{textB, imageB}},+ }, []chat.MessageContentPart{{Type: "text", Text: "compaction summary"}, separator, textB, imageB}},+ {"text after multimodal user", []chat.Message{+ {Role: "user", MultiContent: []chat.MessageContentPart{textB, imageB}},+ {Role: "user", Content: "follow-up"},+ }, []chat.MessageContentPart{textB, imageB, separator, {Type: "text", Text: "follow-up"}}},+ {"mixed representations", []chat.Message{+ {Role: "user", Content: textA.Text, Images: []string{imageA.ImageURL.URL}},+ {Role: "user", Content: "fallback text", MultiContent: []chat.MessageContentPart{textB, imageB}},+ {Role: "user", Content: "compare them"},+ }, []chat.MessageContentPart{imageA, textA, separator, textB, imageB, separator, {+ Type: "text", Text: "compare them",+ }}},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ got := SanitizeMessages(tc.messages)+ require.Len(t, got, 1)+ assert.Equal(t, tc.want, got[0].MultiContent)+ assert.Empty(t, got[0].Images, "images must not be sent twice by adapters reading both representations")+ assert.Empty(t, got[0].Content, "adapters must read all merged text from MultiContent")+ assert.Equal(t, got, SanitizeMessages(got), "sanitizing twice must preserve the payload")+ })+ }+}++func TestSanitizeMessagesDoesNotOverwriteMultimodalBackingArray(t *testing.T) {+ parts := make([]chat.MessageContentPart, 2, 4)+ parts[0] = chat.MessageContentPart{Type: "text", Text: "first"}+ parts[1] = chat.MessageContentPart{Type: "text", Text: "must stay"}+ messages := []chat.Message{+ {Role: "user", MultiContent: parts[:1]},+ {Role: "user", Content: "second"},+ }+ got := SanitizeMessages(messages)+ require.Len(t, got, 1)+ require.Len(t, got[0].MultiContent, 3)+ assert.Equal(t, "must stay", parts[1].Text)+ assert.Equal(t, "first", messages[0].MultiContent[0].Text)+ assert.Equal(t, "second", messages[1].Content)+}diff --git a/internal/agent/tools/sanitize_messages_test.go b/internal/agent/tools/sanitize_messages_test.goindex bcfb7fb0e..d9380a7df 100644--- a/internal/agent/tools/sanitize_messages_test.go+++ b/internal/agent/tools/sanitize_messages_test.go@@ -30,9 +30,21 @@ func TestSanitizeMessages(t *testing.T) {}result := SanitizeMessages(messages)require.Len(t, result, 1)+ var images []string+ var text strings.Builder+ for _, part := range result[0].MultiContent {+ if part.Type == "image_url" && part.ImageURL != nil {+ images = append(images, part.ImageURL.URL)+ }+ if part.Type == "text" {+ text.WriteString(part.Text)+ }+ }assert.Equal(t, []string{"data:image/png;base64,MQ==", "data:image/png;base64,Mg==", "data:image/png;base64,Mw==",- }, result[0].Images)+ }, images)+ assert.Equal(t, "image one\n\nimage two\n\n", text.String())+ assert.Empty(t, result[0].Images, "images are represented once in MultiContent")assert.Len(t, first, 1, "merging must not append into the caller's slice")})