Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(agent): preserve multimodal content during message sanitization (#3949)

澍2f3b04c52e00
diff --git a/internal/agent/compaction/fileops.go b/internal/agent/compaction/fileops.go
index 77060c571..295aaf75b 100644
--- a/internal/agent/compaction/fileops.go
+++ b/internal/agent/compaction/fileops.go
@@ -63,7 +63,7 @@ func extractFileOps(previousSummary string, groups ...[]chat.Message) fileOps {
ops.inherit(previousSummary)
for _, group := range groups {
for _, msg := range group {
- ops.inherit(msg.Content)
+ ops.inherit(messageText(msg))
for _, tc := range msg.ToolCalls {
path := toolCallPath(tc.Function.Arguments)
if path == "" {
diff --git a/internal/agent/compaction/multimodal_overflow_test.go b/internal/agent/compaction/multimodal_overflow_test.go
new file mode 100644
index 000000000..b5d097cdd
--- /dev/null
+++ b/internal/agent/compaction/multimodal_overflow_test.go
@@ -0,0 +1,62 @@
+package compaction
+
+import (
+ "strings"
+ "testing"
+
+ agenttools "github.com/Tencent/WeKnora/internal/agent/tools"
+ "github.com/Tencent/WeKnora/internal/models/chat"
+ "github.com/stretchr/testify/require"
+)
+
+func TestOverflowAfterSanitizationPreservesPreviousSummary(t *testing.T) {
+ const marker = "KEEP_PRIOR_USER_DECISION_7E93"
+ live := reactTurn(40)
+ summary := SummaryMessage(marker + "\n<modified-files>\n/workspace/prior.txt\n</modified-files>")
+ image := chat.Message{Role: "user", MultiContent: []chat.MessageContentPart{
+ {Type: "image_url", ImageURL: &chat.ImageURL{URL: "data:image/png;base64,AA=="}},
+ {Type: "text", Text: "continue from this screenshot"},
+ }}
+ messages := append([]chat.Message{live[0], summary, image}, live[2:]...)
+ sanitized := agenttools.SanitizeMessages(messages)
+ require.Equal(t, chat.MessageKindCompactionSummary, sanitized[1].Kind)
+ require.Empty(t, sanitized[1].Content, "exercise the multipart-only summary produced by sanitization")
+ require.NotEmpty(t, sanitized[1].MultiContent)
+
+ llm := &stubChat{response: "## Goal\ncontinue the task"}
+ c := New(llm, newEstimator(t), testSettings())
+ result, err := c.Compact(t.Context(), sanitized, ReasonOverflow)
+ require.NoError(t, err)
+ require.NotEmpty(t, llm.prompts)
+ require.Contains(t, strings.Join(llm.prompts, "\n"), marker)
+ require.Contains(t, strings.Join(llm.prompts, "\n"), "<previous-summary>")
+ require.Contains(t, result.Summary, "/workspace/prior.txt")
+ require.Equal(t, marker+"\n<modified-files>\n/workspace/prior.txt\n</modified-files>",
+ unwrapSummary(messages[1].Content), "caller-owned messages must not be changed")
+}
+
+func TestCompactionSerializesMultipartTextWithoutLegacyDuplication(t *testing.T) {
+ for _, role := range []string{"user", "assistant", "tool"} {
+ t.Run(role, func(t *testing.T) {
+ message := chat.Message{Role: role, Content: "stale legacy text", MultiContent: []chat.MessageContentPart{
+ {Type: "text", Text: "keep first"},
+ {Type: "image_url", ImageURL: &chat.ImageURL{URL: "https://example.com/image.png"}},
+ {Type: "text", Text: " and second"},
+ }}
+ for _, rendered := range []string{serializeConversation([]chat.Message{message}), archiveLine(&message)} {
+ require.Contains(t, rendered, "keep first and second")
+ require.NotContains(t, rendered, "stale legacy text")
+ require.NotContains(t, rendered, "https://example.com")
+ }
+ })
+ }
+}
+
+func TestCompactionInheritsMultipartFileOperations(t *testing.T) {
+ message := chat.Message{Role: "user", MultiContent: []chat.MessageContentPart{
+ {Type: "text", Text: "<modified-files>\n/workspace/kept.txt\n</modified-files>"},
+ }}
+ ops := extractFileOps("", []chat.Message{message})
+ _, modified := ops.resolve()
+ require.Equal(t, []string{"/workspace/kept.txt"}, modified)
+}
diff --git a/internal/agent/compaction/prepare.go b/internal/agent/compaction/prepare.go
index 18d529ad6..299e4d4a7 100644
--- a/internal/agent/compaction/prepare.go
+++ b/internal/agent/compaction/prepare.go
@@ -66,7 +66,7 @@ func Prepare(messages []chat.Message, s Settings, estimator *agenttoken.Estimato
if messages[i].Kind != chat.MessageKindCompactionSummary {
continue
}
- previousSummary = unwrapSummary(messages[i].Content)
+ previousSummary = unwrapSummary(messageText(messages[i]))
boundaryStart = i + 1
break
}
diff --git a/internal/agent/compaction/serialize.go b/internal/agent/compaction/serialize.go
index f0a6bcab7..2821b911a 100644
--- a/internal/agent/compaction/serialize.go
+++ b/internal/agent/compaction/serialize.go
@@ -31,11 +31,12 @@ func serializeConversation(messages []chat.Message) string {
var parts []string
for i := range messages {
msg := &messages[i]
+ text := messageText(*msg)
switch msg.Role {
case "system":
continue
case "user":
- if content := truncate(msg.Content, textMaxChars); content != "" {
+ if content := truncate(text, textMaxChars); content != "" {
parts = append(parts, "[User]: "+content)
}
case "assistant":
@@ -43,14 +44,14 @@ func serializeConversation(messages []chat.Message) string {
parts = append(parts,
"[Assistant thinking]: "+truncate(msg.ReasoningContent, textMaxChars))
}
- if msg.Content != "" {
- parts = append(parts, "[Assistant]: "+truncate(msg.Content, textMaxChars))
+ if text != "" {
+ parts = append(parts, "[Assistant]: "+truncate(text, textMaxChars))
}
if calls := serializeToolCalls(msg.ToolCalls); calls != "" {
parts = append(parts, "[Assistant tool calls]: "+calls)
}
case "tool":
- if content := truncate(msg.Content, toolResultMaxChars); content != "" {
+ if content := truncate(text, toolResultMaxChars); content != "" {
parts = append(parts, fmt.Sprintf("[Tool result %s]: %s", msg.Name, content))
}
}
@@ -143,16 +144,33 @@ func rawArchive(messages []chat.Message, budget int, estimator *agenttoken.Estim
}
func archiveLine(msg *chat.Message) string {
+ text := messageText(*msg)
switch msg.Role {
case "user":
- return fmt.Sprintf("- User: %s\n", truncate(msg.Content, 500))
+ return fmt.Sprintf("- User: %s\n", truncate(text, 500))
case "assistant":
if calls := serializeToolCalls(msg.ToolCalls); calls != "" {
- return fmt.Sprintf("- Assistant [%s]: %s\n", calls, truncate(msg.Content, 500))
+ return fmt.Sprintf("- Assistant [%s]: %s\n", calls, truncate(text, 500))
}
- return fmt.Sprintf("- Assistant: %s\n", truncate(msg.Content, 500))
+ return fmt.Sprintf("- Assistant: %s\n", truncate(text, 500))
case "tool":
- return fmt.Sprintf("- Tool[%s]: %s\n", msg.Name, truncate(msg.Content, 500))
+ return fmt.Sprintf("- Tool[%s]: %s\n", msg.Name, truncate(text, 500))
}
return ""
}
+
+// messageText follows the providers' preference for multipart content. The
+// sanitizer can move a compaction summary's text into MultiContent while
+// retaining its Kind, so scalar Content alone is not a complete transcript.
+func messageText(msg chat.Message) string {
+ if len(msg.MultiContent) == 0 {
+ return msg.Content
+ }
+ var text strings.Builder
+ for _, part := range msg.MultiContent {
+ if part.Type == "text" {
+ text.WriteString(part.Text)
+ }
+ }
+ return text.String()
+}
diff --git a/internal/agent/resume_test.go b/internal/agent/resume_test.go
index 8f4b6aa3c..0ccfaeff3 100644
--- a/internal/agent/resume_test.go
+++ b/internal/agent/resume_test.go
@@ -152,10 +152,24 @@ func TestResumeReplaysSteeringAtItsOriginalBoundaries(t *testing.T) {
}}}}
_, err = f.engine.Resume(t.Context(), "sess-1", "msg-1", nil, data, answeredPrompt("call-b"))
require.NoError(t, err)
- var contents []string
+ var contents, images []string
for _, m := range model.calls[0] {
- contents = append(contents, m.Content)
+ if len(m.MultiContent) == 0 {
+ contents = append(contents, m.Content)
+ images = append(images, m.Images...)
+ continue
+ }
+ for _, part := range m.MultiContent {
+ switch part.Type {
+ case "text":
+ contents = append(contents, part.Text)
+ case "image_url":
+ require.NotNil(t, part.ImageURL)
+ images = append(images, part.ImageURL.URL)
+ }
+ }
}
+ require.Equal(t, []string{"https://img/1.png"}, images)
// The provider normalization can merge adjacent user messages, including
// the original query and the first steering message.
transcript := strings.Join(contents, "\n")
diff --git a/internal/agent/tools/sanitize_messages.go b/internal/agent/tools/sanitize_messages.go
index f58892f96..3188c57dc 100644
--- a/internal/agent/tools/sanitize_messages.go
+++ b/internal/agent/tools/sanitize_messages.go
@@ -49,8 +49,16 @@ func SanitizeMessages(messages []chat.Message) []chat.Message {
// parts go along for the same reason as tool calls: dropping
// them leaves the model reading captions of images it never got.
merged.ToolCalls = concatCopy(prev.ToolCalls, msg.ToolCalls)
- merged.Images = concatCopy(prev.Images, msg.Images)
- merged.MultiContent = concatCopy(prev.MultiContent, msg.MultiContent)
+ if len(prev.Images)+len(msg.Images)+len(prev.MultiContent)+len(msg.MultiContent) > 0 {
+ // Preserve each message's image/text order in one representation.
+ // Providers prefer MultiContent over Content; retaining only the
+ // latter would lose the appended text and images on the wire.
+ parts := appendSanitizedContent(nil, prev)
+ parts = append(parts, chat.MessageContentPart{Type: "text", Text: "\n\n"})
+ merged.MultiContent = appendSanitizedContent(parts, msg)
+ merged.Content = ""
+ merged.Images = nil
+ }
result[len(result)-1] = merged
continue
}
@@ -82,6 +90,24 @@ func concatCopy[T any](a, b []T) []T {
return append(append(out, a...), b...)
}
+// appendSanitizedContent follows the providers' preference for MultiContent,
+// expanding the legacy Content/Images representation only when needed.
+// The destination is separate from the input messages' backing arrays.
+func appendSanitizedContent(parts []chat.MessageContentPart, msg chat.Message) []chat.MessageContentPart {
+ if len(msg.MultiContent) > 0 {
+ return append(parts, msg.MultiContent...)
+ }
+ for _, url := range msg.Images {
+ parts = append(parts, chat.MessageContentPart{
+ Type: "image_url", ImageURL: &chat.ImageURL{URL: url, Detail: "auto"},
+ })
+ }
+ if msg.Content != "" {
+ parts = append(parts, chat.MessageContentPart{Type: "text", Text: msg.Content})
+ }
+ return parts
+}
+
// hasMatchingToolCall checks if any preceding assistant message has a tool call with the given ID.
func hasMatchingToolCall(messages []chat.Message, toolCallID string) bool {
for i := len(messages) - 1; i >= 0; i-- {
diff --git a/internal/agent/tools/sanitize_messages_multimodal_test.go b/internal/agent/tools/sanitize_messages_multimodal_test.go
new file mode 100644
index 000000000..a45373b92
--- /dev/null
+++ b/internal/agent/tools/sanitize_messages_multimodal_test.go
@@ -0,0 +1,133 @@
+package tools
+
+import (
+ "encoding/json"
+ "testing"
+
+ "github.com/Tencent/WeKnora/internal/models/api"
+ "github.com/Tencent/WeKnora/internal/models/api/openaicompletions"
+ "github.com/Tencent/WeKnora/internal/models/chat"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestSanitizeMessagesPreservesMultimodalContent(t *testing.T) {
+ image := chat.MessageContentPart{
+ Type: "image_url", ImageURL: &chat.ImageURL{URL: "https://example.com/a.png", Detail: "high"},
+ }
+ text := chat.MessageContentPart{Type: "text", Text: "describe this"}
+ for i, msg := range []chat.Message{
+ {Role: "user", Images: []string{"https://example.com/a.png"}},
+ {Role: "user", MultiContent: []chat.MessageContentPart{image}},
+ {Role: "user", MultiContent: []chat.MessageContentPart{text, image}},
+ } {
+ t.Run([]string{"legacy image", "image part", "text and image parts"}[i], func(t *testing.T) {
+ got := SanitizeMessages([]chat.Message{msg})
+ require.Equal(t, []chat.Message{msg}, got, "nonempty multimodal content must not be discarded")
+ })
+ }
+}
+
+func TestSanitizeMessagesKeepsImagesInProviderRequest(t *testing.T) {
+ client := openaicompletions.New(openaicompletions.Config{
+ Endpoint: api.Endpoint{BaseURL: "https://example.com/v1", Model: "vision-model"},
+ Settings: api.DefaultOpenAICompletions(),
+ })
+ for _, stream := range []bool{false, true} {
+ body, err := client.BuildRequestBody(SanitizeMessages([]chat.Message{
+ {Role: "user", Content: "earlier conversation summary"},
+ {Role: "user", MultiContent: []chat.MessageContentPart{
+ {Type: "text", Text: "compare the screenshots"},
+ {Type: "image_url", ImageURL: &chat.ImageURL{URL: "https://example.com/first.png", Detail: "high"}},
+ }},
+ {Role: "user", Content: "second screenshot", Images: []string{"https://example.com/second.png"}},
+ }), nil, stream)
+ require.NoError(t, err)
+ data, err := json.Marshal(body)
+ require.NoError(t, err)
+ var request struct {
+ Messages []struct {
+ Role string `json:"role"`
+ Content json.RawMessage `json:"content"`
+ } `json:"messages"`
+ }
+ require.NoError(t, json.Unmarshal(data, &request))
+ require.Len(t, request.Messages, 1)
+ assert.Equal(t, "user", request.Messages[0].Role)
+ assert.JSONEq(t, `[
+ {"type":"text","text":"earlier conversation summary"},
+ {"type":"text","text":"\n\n"},
+ {"type":"text","text":"compare the screenshots"},
+ {"type":"image_url","image_url":{"url":"https://example.com/first.png","detail":"high"}},
+ {"type":"text","text":"\n\n"},
+ {"type":"image_url","image_url":{"url":"https://example.com/second.png","detail":"auto"}},
+ {"type":"text","text":"second screenshot"}
+ ]`, string(request.Messages[0].Content))
+ }
+}
+
+func TestSanitizeMessagesMergesMultimodalContentInOrder(t *testing.T) {
+ imageA := chat.MessageContentPart{
+ Type: "image_url", ImageURL: &chat.ImageURL{URL: "https://example.com/a.png", Detail: "auto"},
+ }
+ imageB := chat.MessageContentPart{
+ Type: "image_url", ImageURL: &chat.ImageURL{URL: "https://example.com/b.png", Detail: "high"},
+ }
+ legacyB := chat.MessageContentPart{
+ Type: "image_url", ImageURL: &chat.ImageURL{URL: imageB.ImageURL.URL, Detail: "auto"},
+ }
+ textA := chat.MessageContentPart{Type: "text", Text: "first image"}
+ textB := chat.MessageContentPart{Type: "text", Text: "second image"}
+ separator := chat.MessageContentPart{Type: "text", Text: "\n\n"}
+ cases := []struct {
+ name string
+ messages []chat.Message
+ want []chat.MessageContentPart
+ }{
+ {"tool image messages", []chat.Message{
+ {Role: "user", Content: textA.Text, Images: []string{imageA.ImageURL.URL}},
+ {Role: "user", Content: textB.Text, Images: []string{imageB.ImageURL.URL}},
+ }, []chat.MessageContentPart{imageA, textA, separator, legacyB, textB}},
+ {"summary before multimodal user", []chat.Message{
+ {Role: "user", Content: "compaction summary"},
+ {Role: "user", MultiContent: []chat.MessageContentPart{textB, imageB}},
+ }, []chat.MessageContentPart{{Type: "text", Text: "compaction summary"}, separator, textB, imageB}},
+ {"text after multimodal user", []chat.Message{
+ {Role: "user", MultiContent: []chat.MessageContentPart{textB, imageB}},
+ {Role: "user", Content: "follow-up"},
+ }, []chat.MessageContentPart{textB, imageB, separator, {Type: "text", Text: "follow-up"}}},
+ {"mixed representations", []chat.Message{
+ {Role: "user", Content: textA.Text, Images: []string{imageA.ImageURL.URL}},
+ {Role: "user", Content: "fallback text", MultiContent: []chat.MessageContentPart{textB, imageB}},
+ {Role: "user", Content: "compare them"},
+ }, []chat.MessageContentPart{imageA, textA, separator, textB, imageB, separator, {
+ Type: "text", Text: "compare them",
+ }}},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ got := SanitizeMessages(tc.messages)
+ require.Len(t, got, 1)
+ assert.Equal(t, tc.want, got[0].MultiContent)
+ assert.Empty(t, got[0].Images, "images must not be sent twice by adapters reading both representations")
+ assert.Empty(t, got[0].Content, "adapters must read all merged text from MultiContent")
+ assert.Equal(t, got, SanitizeMessages(got), "sanitizing twice must preserve the payload")
+ })
+ }
+}
+
+func TestSanitizeMessagesDoesNotOverwriteMultimodalBackingArray(t *testing.T) {
+ parts := make([]chat.MessageContentPart, 2, 4)
+ parts[0] = chat.MessageContentPart{Type: "text", Text: "first"}
+ parts[1] = chat.MessageContentPart{Type: "text", Text: "must stay"}
+ messages := []chat.Message{
+ {Role: "user", MultiContent: parts[:1]},
+ {Role: "user", Content: "second"},
+ }
+ got := SanitizeMessages(messages)
+ require.Len(t, got, 1)
+ require.Len(t, got[0].MultiContent, 3)
+ assert.Equal(t, "must stay", parts[1].Text)
+ assert.Equal(t, "first", messages[0].MultiContent[0].Text)
+ assert.Equal(t, "second", messages[1].Content)
+}
diff --git a/internal/agent/tools/sanitize_messages_test.go b/internal/agent/tools/sanitize_messages_test.go
index bcfb7fb0e..d9380a7df 100644
--- a/internal/agent/tools/sanitize_messages_test.go
+++ b/internal/agent/tools/sanitize_messages_test.go
@@ -30,9 +30,21 @@ func TestSanitizeMessages(t *testing.T) {
}
result := SanitizeMessages(messages)
require.Len(t, result, 1)
+ var images []string
+ var text strings.Builder
+ for _, part := range result[0].MultiContent {
+ if part.Type == "image_url" && part.ImageURL != nil {
+ images = append(images, part.ImageURL.URL)
+ }
+ if part.Type == "text" {
+ text.WriteString(part.Text)
+ }
+ }
assert.Equal(t, []string{
"data:image/png;base64,MQ==", "data:image/png;base64,Mg==", "data:image/png;base64,Mw==",
- }, result[0].Images)
+ }, images)
+ assert.Equal(t, "image one\n\nimage two\n\n", text.String())
+ assert.Empty(t, result[0].Images, "images are represented once in MultiContent")
assert.Len(t, first, 1, "merging must not append into the caller's slice")
})