RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(go-agent): forward sys.files uploads to vision LLMs (#18246)
euvre3d50f8bcfc62
diff --git a/internal/agent/component/agent.go b/internal/agent/component/agent.goindex 60542569e5..f3e707cc3c 100644--- a/internal/agent/component/agent.go+++ b/internal/agent/component/agent.go@@ -276,19 +276,49 @@ func scanAllStreamForToolCall(_ context.Context, stream *schema.StreamReader[*sc// buildAgentInputMessages assembles the Python-compatible Agent prompt: the// configured history window followed by the current user prompt. The current// in-flight user entry is excluded through SnapshotPriorHistory, because the-// canvas service appends it to state before invoking the workflow.+// canvas service appends it to state before invoking the workflow. Uploaded+// files from sys.files are folded into that user prompt (file texts merged,+// images attached as multi-modal content parts).func buildAgentInputMessages(ctx context.Context, p AgentParam) []*schema.Message {- current := schema.Message{Role: schema.User, Content: p.UserPrompt}- messages := []schema.Message{}- if p.MessageHistoryWindowSize > 0 {- if state, _, err := runtime.GetStateFromContext[*runtime.CanvasState](ctx); err == nil && state != nil {- // Python takes the last 2*N entries from history, which already- // contains the current user input, and then removes that final- // entry before formatting the configured prompt.- priorLimit := p.MessageHistoryWindowSize*2 - 1- messages = prependHistory(messages, state.SnapshotPriorHistory(), priorLimit)+ var state *runtime.CanvasState+ if s, _, err := runtime.GetStateFromContext[*runtime.CanvasState](ctx); err == nil && s != nil {+ state = s+ }+ // Inject sys.files uploads into the current user message, mirroring+ // the LLM component (llm.go) and Python's Agent._prepare_prompt_variables+ // delegation to LLMBundle. Uploaded files land in state.Sys["files"]+ // (service/agent.go) as data:image URIs / parsed text; without this+ // step a vision agent never sees the attached image. File texts merge+ // into the user prompt; images become multi-modal content parts.+ // The {sys.files} placeholder, when present, has already been resolved+ // by ResolveTemplate upstream in invokeNow, so injection here is+ // unconditional — same effective behavior as the LLM component.+ userText := p.UserPrompt+ var images []string+ if state != nil {+ var texts []string+ texts, images = collectSysFiles(state)+ if len(texts) > 0 {+ joined := strings.Join(texts, "\n\n")+ if userText != "" {+ userText += "\n\n" + joined+ } else {+ userText = joined+ }}}+ current := schema.Message{Role: schema.User, Content: userText}+ if len(images) > 0 {+ current = userMessageWithImages(userText, images)+ }+ messages := []schema.Message{}+ if p.MessageHistoryWindowSize > 0 && state != nil {+ // Python takes the last 2*N entries from history, which already+ // contains the current user input, and then removes that final+ // entry before formatting the configured prompt.+ priorLimit := p.MessageHistoryWindowSize*2 - 1+ messages = prependHistory(messages, state.SnapshotPriorHistory(), priorLimit)+ }if len(messages) > 0 && messages[len(messages)-1].Role == current.Role {messages[len(messages)-1] = current} else {diff --git a/internal/agent/component/llm.go b/internal/agent/component/llm.goindex f06a43a8b4..7f5ad882be 100644--- a/internal/agent/component/llm.go+++ b/internal/agent/component/llm.go@@ -887,6 +887,15 @@ func buildMessagesWithImages(system, user string, images []string, cite bool) []return out}+ out = append(out, userMessageWithImages(user, images))+ return out+}++// userMessageWithImages builds a user message carrying the text plus the+// given data-image URIs as eino multi-modal content parts. Shared by the+// LLM component (buildMessagesWithImages) and the Agent component+// (buildAgentInputMessages) so both produce the exact same part shape.+func userMessageWithImages(user string, images []string) schema.Message {parts := make([]schema.MessageInputPart, 0, 1+len(images))if user != "" {parts = append(parts, schema.MessageInputPart{@@ -903,11 +912,10 @@ func buildMessagesWithImages(system, user string, images []string, cite bool) []},})}- out = append(out, schema.Message{+ return schema.Message{Role: schema.User,UserInputMultiContent: parts,- })- return out+ }}// mergeLLMParam layers raw inputs over the receiver's default param set.diff --git a/internal/agent/component/vision_test.go b/internal/agent/component/vision_test.goindex c0ba352084..02d43bcd4c 100644--- a/internal/agent/component/vision_test.go+++ b/internal/agent/component/vision_test.go@@ -373,3 +373,58 @@ func TestLLM_Invoke_VisualFilesAsString(t *testing.T) {user.UserInputMultiContent[1])}}++// TestBuildAgentInputMessagesInjectsSysFiles guards the agent-side half of+// the upload fix: buildAgentInputMessages must fold sys.files uploads into+// the current user message (file text merged, images as multi-modal parts).+// Without it a vision agent (e.g. qwen3-vl-plus) reports it cannot see the+// attached image.+func TestBuildAgentInputMessagesInjectsSysFiles(t *testing.T) {+ uri := "data:image/png;base64,iVBORw0KGgo="+ state := runtime.NewCanvasState("run-agent-files", "task-agent-files")+ state.Sys["files"] = []string{"parsed document text", uri}+ ctx := runtime.WithState(t.Context(), state)++ messages := buildAgentInputMessages(ctx, AgentParam{UserPrompt: "描述图片内容"})+ if len(messages) != 1 {+ t.Fatalf("message count = %d, want 1", len(messages))+ }+ msg := messages[0]+ if msg.Role != schema.User {+ t.Fatalf("role = %v, want user", msg.Role)+ }+ if len(msg.UserInputMultiContent) != 2 {+ t.Fatalf("parts = %d, want 2 (text + image)", len(msg.UserInputMultiContent))+ }+ textPart := msg.UserInputMultiContent[0]+ if textPart.Type != schema.ChatMessagePartTypeText ||+ textPart.Text != "描述图片内容\n\nparsed document text" {+ t.Errorf("text part = %+v, want merged prompt+file text", textPart)+ }+ imagePart := msg.UserInputMultiContent[1]+ if imagePart.Type != schema.ChatMessagePartTypeImageURL ||+ imagePart.Image == nil || imagePart.Image.URL == nil || *imagePart.Image.URL != uri {+ t.Errorf("image part = %+v, want image_url part carrying the upload URI", imagePart)+ }+}++// TestBuildAgentInputMessagesSysFilesTextOnly: non-image uploads merge into+// the plain string Content without creating multi-modal parts.+func TestBuildAgentInputMessagesSysFilesTextOnly(t *testing.T) {+ state := runtime.NewCanvasState("run-agent-text", "task-agent-text")+ state.Sys["files"] = []string{"first file text", "second file text"}+ ctx := runtime.WithState(t.Context(), state)++ messages := buildAgentInputMessages(ctx, AgentParam{UserPrompt: "summarize"})+ if len(messages) != 1 {+ t.Fatalf("message count = %d, want 1", len(messages))+ }+ want := "summarize\n\nfirst file text\n\nsecond file text"+ if messages[0].Content != want {+ t.Errorf("Content = %q, want %q", messages[0].Content, want)+ }+ if len(messages[0].UserInputMultiContent) != 0 {+ t.Errorf("text-only files must not create multi-modal parts, got %d",+ len(messages[0].UserInputMultiContent))+ }+}diff --git a/internal/entity/models/llm.go b/internal/entity/models/llm.goindex 13bd54cfb9..d3cdecf3ea 100644--- a/internal/entity/models/llm.go+++ b/internal/entity/models/llm.go@@ -91,6 +91,11 @@ func toInternalMessages(msgs []*schema.Message) []Message {role = "user"}msg := Message{Role: role, Content: mm.Content}+ if len(mm.UserInputMultiContent) > 0 {+ if blocks := openAIContentBlocksFromEino(mm.UserInputMultiContent); len(blocks) > 0 {+ msg.Content = blocks+ }+ }if len(mm.ToolCalls) > 0 {msg.ToolCalls = toolCallsToInternal(mm.ToolCalls)}@@ -102,6 +107,62 @@ func toInternalMessages(msgs []*schema.Message) []Message {return out}+// openAIContentBlocksFromEino converts eino multi-modal input parts into+// OpenAI-style content blocks ("text" / "image_url"). Message.Content is+// interface{} and every driver already understands this block shape: the+// generic OpenAI-compatible request builder marshals it verbatim+// (buildChatMessages in base_model.go), while the native anthropic /+// google converters type-switch on []interface{} (anthropicContent /+// googleMessageParts). The slice MUST therefore be []interface{}, not+// []map[string]interface{}, or googleMessageParts misses it. Unsupported+// part types are skipped; a nil return tells the caller to fall back to+// the plain string Content.+func openAIContentBlocksFromEino(parts []schema.MessageInputPart) []interface{} {+ blocks := make([]interface{}, 0, len(parts))+ for _, part := range parts {+ switch part.Type {+ case schema.ChatMessagePartTypeText:+ if part.Text == "" {+ continue+ }+ blocks = append(blocks, map[string]interface{}{"type": "text", "text": part.Text})+ case schema.ChatMessagePartTypeImageURL:+ url := einoImagePartURL(part.Image)+ if url == "" {+ continue+ }+ blocks = append(blocks, map[string]interface{}{+ "type": "image_url",+ "image_url": map[string]interface{}{"url": url},+ })+ }+ }+ if len(blocks) == 0 {+ return nil+ }+ return blocks+}++// einoImagePartURL resolves an image part to a single URL string: either+// the direct URL (the agent component carries data URIs this way) or a+// reassembled data URI from Base64Data + MIMEType.+func einoImagePartURL(img *schema.MessageInputImage) string {+ if img == nil {+ return ""+ }+ if img.URL != nil && *img.URL != "" {+ return *img.URL+ }+ if img.Base64Data != nil && *img.Base64Data != "" {+ mime := img.MIMEType+ if mime == "" {+ mime = "image/png"+ }+ return "data:" + mime + ";base64," + *img.Base64Data+ }+ return ""+}+// fromInternalResponse converts a *ChatResponse to *schema.Message. The// existing ChatResponse only carries answer text (+ optional reasoning), so// the resulting Message has Role=Assistant and Content=answer.diff --git a/internal/entity/models/llm_test.go b/internal/entity/models/llm_test.goindex 7e350c1226..723831c4b4 100644--- a/internal/entity/models/llm_test.go+++ b/internal/entity/models/llm_test.go@@ -301,3 +301,95 @@ func (d *captureToolDriver) ListTasks(ctx context.Context, _ *APIConfig) ([]Listfunc (d *captureToolDriver) ShowTask(ctx context.Context, _ string, _ *APIConfig) (*TaskResponse, error) {return nil, nil}++// TestToInternalMessagesConvertsMultiModalContent guards the eino→driver+// boundary: UserInputMultiContent must become OpenAI-style content blocks+// ([]interface{} of {type:text} / {type:image_url}) on Message.Content,+// otherwise image parts produced by the component layer are silently+// dropped before the request reaches any driver.+func TestToInternalMessagesConvertsMultiModalContent(t *testing.T) {+ uri := "data:image/png;base64,iVBORw0KGgo="+ internal := toInternalMessages([]*schema.Message{+ {+ Role: schema.User,+ UserInputMultiContent: []schema.MessageInputPart{+ {Type: schema.ChatMessagePartTypeText, Text: "describe the image"},+ {Type: schema.ChatMessagePartTypeImageURL,+ Image: &schema.MessageInputImage{+ MessagePartCommon: schema.MessagePartCommon{URL: &uri},+ }},+ },+ },+ })+ if len(internal) != 1 {+ t.Fatalf("len(internal) = %d, want 1", len(internal))+ }+ blocks, ok := internal[0].Content.([]interface{})+ if !ok {+ t.Fatalf("Content type = %T, want []interface{} content blocks", internal[0].Content)+ }+ if len(blocks) != 2 {+ t.Fatalf("len(blocks) = %d, want 2", len(blocks))+ }+ textBlock, ok := blocks[0].(map[string]interface{})+ if !ok || textBlock["type"] != "text" || textBlock["text"] != "describe the image" {+ t.Fatalf("text block = %#v, want {type:text, text:describe the image}", blocks[0])+ }+ imageBlock, ok := blocks[1].(map[string]interface{})+ if !ok || imageBlock["type"] != "image_url" {+ t.Fatalf("image block = %#v, want type image_url", blocks[1])+ }+ imageURL, ok := imageBlock["image_url"].(map[string]interface{})+ if !ok || imageURL["url"] != uri {+ t.Fatalf("image_url = %#v, want url %q", imageBlock["image_url"], uri)+ }+}++// TestToInternalMessagesReassemblesBase64Image: parts that carry Base64Data+// instead of a URL are reassembled into a data URI.+func TestToInternalMessagesReassemblesBase64Image(t *testing.T) {+ b64 := "aGVsbG8="+ internal := toInternalMessages([]*schema.Message{+ {+ Role: schema.User,+ UserInputMultiContent: []schema.MessageInputPart{+ {Type: schema.ChatMessagePartTypeImageURL,+ Image: &schema.MessageInputImage{+ MessagePartCommon: schema.MessagePartCommon{+ Base64Data: &b64,+ MIMEType: "image/jpeg",+ },+ }},+ },+ },+ })+ blocks, ok := internal[0].Content.([]interface{})+ if !ok || len(blocks) != 1 {+ t.Fatalf("Content = %#v, want one content block", internal[0].Content)+ }+ imageBlock, ok := blocks[0].(map[string]interface{})+ if !ok {+ t.Fatalf("block = %#v, want map", blocks[0])+ }+ imageURL, ok := imageBlock["image_url"].(map[string]interface{})+ if !ok || imageURL["url"] != "data:image/jpeg;base64,aGVsbG8=" {+ t.Fatalf("image_url = %#v, want reassembled data URI", imageBlock["image_url"])+ }+}++// TestToInternalMessagesUnsupportedPartsFallBackToString: when every part is+// of an unsupported type, Content stays the plain string.+func TestToInternalMessagesUnsupportedPartsFallBackToString(t *testing.T) {+ internal := toInternalMessages([]*schema.Message{+ {+ Role: schema.User,+ Content: "plain",+ UserInputMultiContent: []schema.MessageInputPart{+ {Type: schema.ChatMessagePartTypeAudioURL},+ },+ },+ })+ if content, ok := internal[0].Content.(string); !ok || content != "plain" {+ t.Fatalf("Content = %#v, want string %q", internal[0].Content, "plain")+ }+}