Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(go-agent): forward sys.files uploads to vision LLMs (#18246)

euvre3d50f8bcfc62
diff --git a/internal/agent/component/agent.go b/internal/agent/component/agent.go
index 60542569e5..f3e707cc3c 100644
--- a/internal/agent/component/agent.go
+++ b/internal/agent/component/agent.go
@@ -276,19 +276,49 @@ func scanAllStreamForToolCall(_ context.Context, stream *schema.StreamReader[*sc
// buildAgentInputMessages assembles the Python-compatible Agent prompt: the
// configured history window followed by the current user prompt. The current
// in-flight user entry is excluded through SnapshotPriorHistory, because the
-// canvas service appends it to state before invoking the workflow.
+// canvas service appends it to state before invoking the workflow. Uploaded
+// files from sys.files are folded into that user prompt (file texts merged,
+// images attached as multi-modal content parts).
func buildAgentInputMessages(ctx context.Context, p AgentParam) []*schema.Message {
- current := schema.Message{Role: schema.User, Content: p.UserPrompt}
- messages := []schema.Message{}
- if p.MessageHistoryWindowSize > 0 {
- if state, _, err := runtime.GetStateFromContext[*runtime.CanvasState](ctx); err == nil && state != nil {
- // Python takes the last 2*N entries from history, which already
- // contains the current user input, and then removes that final
- // entry before formatting the configured prompt.
- priorLimit := p.MessageHistoryWindowSize*2 - 1
- messages = prependHistory(messages, state.SnapshotPriorHistory(), priorLimit)
+ var state *runtime.CanvasState
+ if s, _, err := runtime.GetStateFromContext[*runtime.CanvasState](ctx); err == nil && s != nil {
+ state = s
+ }
+ // Inject sys.files uploads into the current user message, mirroring
+ // the LLM component (llm.go) and Python's Agent._prepare_prompt_variables
+ // delegation to LLMBundle. Uploaded files land in state.Sys["files"]
+ // (service/agent.go) as data:image URIs / parsed text; without this
+ // step a vision agent never sees the attached image. File texts merge
+ // into the user prompt; images become multi-modal content parts.
+ // The {sys.files} placeholder, when present, has already been resolved
+ // by ResolveTemplate upstream in invokeNow, so injection here is
+ // unconditional — same effective behavior as the LLM component.
+ userText := p.UserPrompt
+ var images []string
+ if state != nil {
+ var texts []string
+ texts, images = collectSysFiles(state)
+ if len(texts) > 0 {
+ joined := strings.Join(texts, "\n\n")
+ if userText != "" {
+ userText += "\n\n" + joined
+ } else {
+ userText = joined
+ }
}
}
+ current := schema.Message{Role: schema.User, Content: userText}
+ if len(images) > 0 {
+ current = userMessageWithImages(userText, images)
+ }
+ messages := []schema.Message{}
+ if p.MessageHistoryWindowSize > 0 && state != nil {
+ // Python takes the last 2*N entries from history, which already
+ // contains the current user input, and then removes that final
+ // entry before formatting the configured prompt.
+ priorLimit := p.MessageHistoryWindowSize*2 - 1
+ messages = prependHistory(messages, state.SnapshotPriorHistory(), priorLimit)
+ }
if len(messages) > 0 && messages[len(messages)-1].Role == current.Role {
messages[len(messages)-1] = current
} else {
diff --git a/internal/agent/component/llm.go b/internal/agent/component/llm.go
index f06a43a8b4..7f5ad882be 100644
--- a/internal/agent/component/llm.go
+++ b/internal/agent/component/llm.go
@@ -887,6 +887,15 @@ func buildMessagesWithImages(system, user string, images []string, cite bool) []
return out
}
+ out = append(out, userMessageWithImages(user, images))
+ return out
+}
+
+// userMessageWithImages builds a user message carrying the text plus the
+// given data-image URIs as eino multi-modal content parts. Shared by the
+// LLM component (buildMessagesWithImages) and the Agent component
+// (buildAgentInputMessages) so both produce the exact same part shape.
+func userMessageWithImages(user string, images []string) schema.Message {
parts := make([]schema.MessageInputPart, 0, 1+len(images))
if user != "" {
parts = append(parts, schema.MessageInputPart{
@@ -903,11 +912,10 @@ func buildMessagesWithImages(system, user string, images []string, cite bool) []
},
})
}
- out = append(out, schema.Message{
+ return schema.Message{
Role: schema.User,
UserInputMultiContent: parts,
- })
- return out
+ }
}
// mergeLLMParam layers raw inputs over the receiver's default param set.
diff --git a/internal/agent/component/vision_test.go b/internal/agent/component/vision_test.go
index c0ba352084..02d43bcd4c 100644
--- a/internal/agent/component/vision_test.go
+++ b/internal/agent/component/vision_test.go
@@ -373,3 +373,58 @@ func TestLLM_Invoke_VisualFilesAsString(t *testing.T) {
user.UserInputMultiContent[1])
}
}
+
+// TestBuildAgentInputMessagesInjectsSysFiles guards the agent-side half of
+// the upload fix: buildAgentInputMessages must fold sys.files uploads into
+// the current user message (file text merged, images as multi-modal parts).
+// Without it a vision agent (e.g. qwen3-vl-plus) reports it cannot see the
+// attached image.
+func TestBuildAgentInputMessagesInjectsSysFiles(t *testing.T) {
+ uri := "data:image/png;base64,iVBORw0KGgo="
+ state := runtime.NewCanvasState("run-agent-files", "task-agent-files")
+ state.Sys["files"] = []string{"parsed document text", uri}
+ ctx := runtime.WithState(t.Context(), state)
+
+ messages := buildAgentInputMessages(ctx, AgentParam{UserPrompt: "描述图片内容"})
+ if len(messages) != 1 {
+ t.Fatalf("message count = %d, want 1", len(messages))
+ }
+ msg := messages[0]
+ if msg.Role != schema.User {
+ t.Fatalf("role = %v, want user", msg.Role)
+ }
+ if len(msg.UserInputMultiContent) != 2 {
+ t.Fatalf("parts = %d, want 2 (text + image)", len(msg.UserInputMultiContent))
+ }
+ textPart := msg.UserInputMultiContent[0]
+ if textPart.Type != schema.ChatMessagePartTypeText ||
+ textPart.Text != "描述图片内容\n\nparsed document text" {
+ t.Errorf("text part = %+v, want merged prompt+file text", textPart)
+ }
+ imagePart := msg.UserInputMultiContent[1]
+ if imagePart.Type != schema.ChatMessagePartTypeImageURL ||
+ imagePart.Image == nil || imagePart.Image.URL == nil || *imagePart.Image.URL != uri {
+ t.Errorf("image part = %+v, want image_url part carrying the upload URI", imagePart)
+ }
+}
+
+// TestBuildAgentInputMessagesSysFilesTextOnly: non-image uploads merge into
+// the plain string Content without creating multi-modal parts.
+func TestBuildAgentInputMessagesSysFilesTextOnly(t *testing.T) {
+ state := runtime.NewCanvasState("run-agent-text", "task-agent-text")
+ state.Sys["files"] = []string{"first file text", "second file text"}
+ ctx := runtime.WithState(t.Context(), state)
+
+ messages := buildAgentInputMessages(ctx, AgentParam{UserPrompt: "summarize"})
+ if len(messages) != 1 {
+ t.Fatalf("message count = %d, want 1", len(messages))
+ }
+ want := "summarize\n\nfirst file text\n\nsecond file text"
+ if messages[0].Content != want {
+ t.Errorf("Content = %q, want %q", messages[0].Content, want)
+ }
+ if len(messages[0].UserInputMultiContent) != 0 {
+ t.Errorf("text-only files must not create multi-modal parts, got %d",
+ len(messages[0].UserInputMultiContent))
+ }
+}
diff --git a/internal/entity/models/llm.go b/internal/entity/models/llm.go
index 13bd54cfb9..d3cdecf3ea 100644
--- a/internal/entity/models/llm.go
+++ b/internal/entity/models/llm.go
@@ -91,6 +91,11 @@ func toInternalMessages(msgs []*schema.Message) []Message {
role = "user"
}
msg := Message{Role: role, Content: mm.Content}
+ if len(mm.UserInputMultiContent) > 0 {
+ if blocks := openAIContentBlocksFromEino(mm.UserInputMultiContent); len(blocks) > 0 {
+ msg.Content = blocks
+ }
+ }
if len(mm.ToolCalls) > 0 {
msg.ToolCalls = toolCallsToInternal(mm.ToolCalls)
}
@@ -102,6 +107,62 @@ func toInternalMessages(msgs []*schema.Message) []Message {
return out
}
+// openAIContentBlocksFromEino converts eino multi-modal input parts into
+// OpenAI-style content blocks ("text" / "image_url"). Message.Content is
+// interface{} and every driver already understands this block shape: the
+// generic OpenAI-compatible request builder marshals it verbatim
+// (buildChatMessages in base_model.go), while the native anthropic /
+// google converters type-switch on []interface{} (anthropicContent /
+// googleMessageParts). The slice MUST therefore be []interface{}, not
+// []map[string]interface{}, or googleMessageParts misses it. Unsupported
+// part types are skipped; a nil return tells the caller to fall back to
+// the plain string Content.
+func openAIContentBlocksFromEino(parts []schema.MessageInputPart) []interface{} {
+ blocks := make([]interface{}, 0, len(parts))
+ for _, part := range parts {
+ switch part.Type {
+ case schema.ChatMessagePartTypeText:
+ if part.Text == "" {
+ continue
+ }
+ blocks = append(blocks, map[string]interface{}{"type": "text", "text": part.Text})
+ case schema.ChatMessagePartTypeImageURL:
+ url := einoImagePartURL(part.Image)
+ if url == "" {
+ continue
+ }
+ blocks = append(blocks, map[string]interface{}{
+ "type": "image_url",
+ "image_url": map[string]interface{}{"url": url},
+ })
+ }
+ }
+ if len(blocks) == 0 {
+ return nil
+ }
+ return blocks
+}
+
+// einoImagePartURL resolves an image part to a single URL string: either
+// the direct URL (the agent component carries data URIs this way) or a
+// reassembled data URI from Base64Data + MIMEType.
+func einoImagePartURL(img *schema.MessageInputImage) string {
+ if img == nil {
+ return ""
+ }
+ if img.URL != nil && *img.URL != "" {
+ return *img.URL
+ }
+ if img.Base64Data != nil && *img.Base64Data != "" {
+ mime := img.MIMEType
+ if mime == "" {
+ mime = "image/png"
+ }
+ return "data:" + mime + ";base64," + *img.Base64Data
+ }
+ return ""
+}
+
// fromInternalResponse converts a *ChatResponse to *schema.Message. The
// existing ChatResponse only carries answer text (+ optional reasoning), so
// the resulting Message has Role=Assistant and Content=answer.
diff --git a/internal/entity/models/llm_test.go b/internal/entity/models/llm_test.go
index 7e350c1226..723831c4b4 100644
--- a/internal/entity/models/llm_test.go
+++ b/internal/entity/models/llm_test.go
@@ -301,3 +301,95 @@ func (d *captureToolDriver) ListTasks(ctx context.Context, _ *APIConfig) ([]List
func (d *captureToolDriver) ShowTask(ctx context.Context, _ string, _ *APIConfig) (*TaskResponse, error) {
return nil, nil
}
+
+// TestToInternalMessagesConvertsMultiModalContent guards the eino→driver
+// boundary: UserInputMultiContent must become OpenAI-style content blocks
+// ([]interface{} of {type:text} / {type:image_url}) on Message.Content,
+// otherwise image parts produced by the component layer are silently
+// dropped before the request reaches any driver.
+func TestToInternalMessagesConvertsMultiModalContent(t *testing.T) {
+ uri := "data:image/png;base64,iVBORw0KGgo="
+ internal := toInternalMessages([]*schema.Message{
+ {
+ Role: schema.User,
+ UserInputMultiContent: []schema.MessageInputPart{
+ {Type: schema.ChatMessagePartTypeText, Text: "describe the image"},
+ {Type: schema.ChatMessagePartTypeImageURL,
+ Image: &schema.MessageInputImage{
+ MessagePartCommon: schema.MessagePartCommon{URL: &uri},
+ }},
+ },
+ },
+ })
+ if len(internal) != 1 {
+ t.Fatalf("len(internal) = %d, want 1", len(internal))
+ }
+ blocks, ok := internal[0].Content.([]interface{})
+ if !ok {
+ t.Fatalf("Content type = %T, want []interface{} content blocks", internal[0].Content)
+ }
+ if len(blocks) != 2 {
+ t.Fatalf("len(blocks) = %d, want 2", len(blocks))
+ }
+ textBlock, ok := blocks[0].(map[string]interface{})
+ if !ok || textBlock["type"] != "text" || textBlock["text"] != "describe the image" {
+ t.Fatalf("text block = %#v, want {type:text, text:describe the image}", blocks[0])
+ }
+ imageBlock, ok := blocks[1].(map[string]interface{})
+ if !ok || imageBlock["type"] != "image_url" {
+ t.Fatalf("image block = %#v, want type image_url", blocks[1])
+ }
+ imageURL, ok := imageBlock["image_url"].(map[string]interface{})
+ if !ok || imageURL["url"] != uri {
+ t.Fatalf("image_url = %#v, want url %q", imageBlock["image_url"], uri)
+ }
+}
+
+// TestToInternalMessagesReassemblesBase64Image: parts that carry Base64Data
+// instead of a URL are reassembled into a data URI.
+func TestToInternalMessagesReassemblesBase64Image(t *testing.T) {
+ b64 := "aGVsbG8="
+ internal := toInternalMessages([]*schema.Message{
+ {
+ Role: schema.User,
+ UserInputMultiContent: []schema.MessageInputPart{
+ {Type: schema.ChatMessagePartTypeImageURL,
+ Image: &schema.MessageInputImage{
+ MessagePartCommon: schema.MessagePartCommon{
+ Base64Data: &b64,
+ MIMEType: "image/jpeg",
+ },
+ }},
+ },
+ },
+ })
+ blocks, ok := internal[0].Content.([]interface{})
+ if !ok || len(blocks) != 1 {
+ t.Fatalf("Content = %#v, want one content block", internal[0].Content)
+ }
+ imageBlock, ok := blocks[0].(map[string]interface{})
+ if !ok {
+ t.Fatalf("block = %#v, want map", blocks[0])
+ }
+ imageURL, ok := imageBlock["image_url"].(map[string]interface{})
+ if !ok || imageURL["url"] != "data:image/jpeg;base64,aGVsbG8=" {
+ t.Fatalf("image_url = %#v, want reassembled data URI", imageBlock["image_url"])
+ }
+}
+
+// TestToInternalMessagesUnsupportedPartsFallBackToString: when every part is
+// of an unsupported type, Content stays the plain string.
+func TestToInternalMessagesUnsupportedPartsFallBackToString(t *testing.T) {
+ internal := toInternalMessages([]*schema.Message{
+ {
+ Role: schema.User,
+ Content: "plain",
+ UserInputMultiContent: []schema.MessageInputPart{
+ {Type: schema.ChatMessagePartTypeAudioURL},
+ },
+ },
+ })
+ if content, ok := internal[0].Content.(string); !ok || content != "plain" {
+ t.Fatalf("Content = %#v, want string %q", internal[0].Content, "plain")
+ }
+}