Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(chat): deliver uploaded image attachments to vision models in Go chat pipeline (#18708)

euvre874d156c5734
diff --git a/internal/service/chat_pipeline.go b/internal/service/chat_pipeline.go
index 2dde6f6cb9..bca2dfb4d8 100644
--- a/internal/service/chat_pipeline.go
+++ b/internal/service/chat_pipeline.go
@@ -1367,13 +1367,18 @@ func (s *ChatPipelineService) AsyncChatSolo(
isImage2Text := modelType == "image2text"
if len(messages) > 0 {
if files, hasFiles := messages[len(messages)-1]["files"]; hasFiles {
- attachmentsStr = s.processFileAttachments(ctx, userID, files)
+ var images []string
+ attachmentsStr, images = s.splitChatAttachments(ctx, userID, files)
if isImage2Text {
- imageFiles = s.extractRawImageURLs(files)
+ imageFiles = images
} else {
common.Debug("AsyncChatSolo: dropping image attachments for text-only chat model",
zap.String("llm_id", chat.LLMID))
}
+ common.Info("AsyncChatSolo: file attachments resolved",
+ zap.Bool("vision_model", isImage2Text),
+ zap.Int("image_files", len(imageFiles)),
+ zap.Int("text_attachment_bytes", len(attachmentsStr)))
}
}
@@ -1623,53 +1628,71 @@ func (s *ChatPipelineService) AsyncChatSolo(
return out, nil
}
-// extractRawImageURLs extracts image references as raw URLs/data-URIs from
-// the string-mode files list, WITHOUT fetching blobs and WITHOUT filtering
-// to data: prefixes. Used for image2text models that expect URLs in the
-// multimodal content (matches Python's `image_files` from
-// `split_file_attachments(files, raw=True)` at
-// dialog_service.py:371-392).
+// splitChatAttachments resolves the last message's file attachments into
+// text content (entries joined by "\n\n") and image data URIs, reading
+// each blob from storage at most once.
//
-// The downstream ConvertLastUserMsgToMultimodal calls parseDataURIOrB64
-// (multimodal.go:63-92) which correctly handles all three forms:
-// - data: URI → base64 source
-// - http:// or https:// URL → URL source
-// - raw base64 → base64 source (default media type)
+// - File-dict mode (the chat UI's upload_info flow): FileService.
+// GetFileContents fetches the blobs; non-visual files are parsed to
+// text, visual files come back as base64 data URIs.
+// - String mode (pre-resolved content): data:-prefixed entries become
+// images; the remaining non-empty entries become text.
//
-// File-dict mode is a known limitation: returns empty for now. A future
-// FileService.GetFileURLsForChat (mirror of GetFileContents with
-// raw=true) would be needed to fully cover the file-dict + image2text
-// combination. The Python equivalent has the same limitation
-// (split_file_attachments calls FileService.get_files which doesn't
-// fetch blobs in raw mode).
-func (s *ChatPipelineService) extractRawImageURLs(files interface{}) []string {
+// Downstream ConvertLastUserMsgToMultimodal → parseDataURIOrB64 accepts the
+// returned data URIs directly.
+func (s *ChatPipelineService) splitChatAttachments(ctx context.Context, userID string, files interface{}) (string, []string) {
+ // ── File-dict mode ──
if fileDicts, ok := parseFileDicts(files); ok {
- _ = fileDicts // see file-dict limitation comment above
- common.Debug("AsyncChatSolo: file-dict + image2text not yet supported; image refs dropped",
- zap.Int("file_dict_count", len(fileDicts)))
- return nil
+ // Only used for GetFileContents (read-only); nil DocRemover means
+ // this FileService MUST NOT be used for DeleteFiles.
+ fileSvc := file.NewFileService(CheckFileTeamPermission, nil)
+ texts, images, err := fileSvc.GetFileContents(ctx, userID, fileDicts)
+ if err != nil {
+ common.Warn("GetFileContents failed in splitChatAttachments",
+ zap.Error(err))
+ return "", nil
+ }
+ if len(texts) == 0 {
+ return "", images
+ }
+ return strings.Join(texts, "\n\n"), images
}
- // String-mode: return all entries as-is. The downstream
- // ConvertLastUserMsgToMultimodal + parseDataURIOrB64 will
- // dispatch on prefix (data: → base64, http(s): → url, else →
- // raw base64).
- var urls []string
+ // ── String mode ──
+ var texts []string
+ var images []string
switch v := files.(type) {
case []string:
for _, f := range v {
- if f != "" {
- urls = append(urls, f)
+ if f = strings.TrimSpace(f); f == "" {
+ continue
+ }
+ if strings.HasPrefix(f, "data:") {
+ images = append(images, f)
+ } else {
+ texts = append(texts, f)
}
}
case []interface{}:
for _, f := range v {
- if s, ok := f.(string); ok && s != "" {
- urls = append(urls, s)
+ str, ok := f.(string)
+ if !ok {
+ continue
+ }
+ if str = strings.TrimSpace(str); str == "" {
+ continue
+ }
+ if strings.HasPrefix(str, "data:") {
+ images = append(images, str)
+ } else {
+ texts = append(texts, str)
}
}
}
- return urls
+ if len(texts) == 0 {
+ return "", images
+ }
+ return strings.Join(texts, "\n\n"), images
}
// ---------------------------------------------------------------------------
@@ -2070,53 +2093,6 @@ func lastUserQuestion(messages []map[string]interface{}) string {
return ""
}
-// processFileAttachments extracts text content from file attachments.
-// Mirrors Python's split_file_attachments (dialog_service.py:371-392)
-// in raw=false mode: returns text attachments joined by "\n\n",
-// filtering out data-URI image attachments.
-//
-// When files are file dicts (Python-compatible format), calls
-// FileService.GetFileContents to fetch actual blobs from storage.
-func (s *ChatPipelineService) processFileAttachments(ctx context.Context, userID string, files interface{}) string {
- // ── File-dict mode ──
- if fileDicts, ok := parseFileDicts(files); ok {
- // Only used for GetFileContents (read-only); nil DocRemover means
- // this FileService MUST NOT be used for DeleteFiles.
- fileSvc := file.NewFileService(CheckFileTeamPermission, nil)
- texts, _, err := fileSvc.GetFileContents(ctx, userID, fileDicts, false)
- if err != nil {
- common.Warn("GetFileContents failed in processFileAttachments",
- zap.Error(err))
- return ""
- }
- if len(texts) == 0 {
- return ""
- }
- return strings.Join(texts, "\n\n")
- }
-
- // ── String fallback ──
- var texts []string
- switch v := files.(type) {
- case []string:
- for _, f := range v {
- if s := strings.TrimSpace(f); s != "" && !strings.HasPrefix(s, "data:") {
- texts = append(texts, s)
- }
- }
- case []interface{}:
- for _, f := range v {
- if s, ok := f.(string); ok && strings.TrimSpace(s) != "" && !strings.HasPrefix(s, "data:") {
- texts = append(texts, s)
- }
- }
- }
- if len(texts) == 0 {
- return ""
- }
- return strings.Join(texts, "\n\n")
-}
-
// splitFileAttachments mirrors Python's `split_file_attachments` at
// dialog_service.py:371-392. It separates `messages[-1]["files"]`
// into text-file content and image attachments.
@@ -2125,8 +2101,8 @@ func (s *ChatPipelineService) processFileAttachments(ctx context.Context, userID
//
// 1. File-dict mode: When `files` is `[]map[string]interface{}` (each dict
// with keys "id", "created_by", "mime_type", "name"), the method calls
-// FileService.GetFileContents to fetch actual file blobs from
-// storage, mirroring Python's FileService.get_files().
+// FileService.GetFileContents to fetch actual file blobs from storage;
+// images come back as base64 data URIs.
//
// 2. String-fallback mode: When `files` is `[]string` or `[]interface{}` of
// strings (pre-resolved content), the method does simple string splitting:
@@ -2140,7 +2116,7 @@ func splitFileAttachments(ctx context.Context, userID string, files interface{},
// Only used for GetFileContents (read-only); nil DocRemover means
// this FileService MUST NOT be used for DeleteFiles.
fileSvc := file.NewFileService(CheckFileTeamPermission, nil)
- texts, images, err := fileSvc.GetFileContents(ctx, userID, fileDicts, raw)
+ texts, images, err := fileSvc.GetFileContents(ctx, userID, fileDicts)
if err != nil {
common.Warn("GetFileContents failed, falling back to string splitting",
zap.Error(err))
diff --git a/internal/service/chat_pipeline_image_test.go b/internal/service/chat_pipeline_image_test.go
new file mode 100644
index 0000000000..6aa7d479b7
--- /dev/null
+++ b/internal/service/chat_pipeline_image_test.go
@@ -0,0 +1,170 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package service
+
+import (
+ "context"
+ "encoding/base64"
+ "strings"
+ "testing"
+
+ "ragflow/internal/common"
+ "ragflow/internal/storage"
+)
+
+// seedChatImageStorage installs an in-memory storage holding one jpeg blob
+// under user-1's downloads bucket, the layout the upload_info flow writes.
+func seedChatImageStorage(t *testing.T) {
+ t.Helper()
+ memory := storage.NewMemoryStorage()
+ if err := memory.Put(t.Context(), "user-1-downloads", "img-1", []byte("jpeg-bytes")); err != nil {
+ t.Fatalf("put: %v", err)
+ }
+ factory := storage.GetStorageFactory()
+ original := factory.GetStorage()
+ factory.SetStorage(memory)
+ t.Cleanup(func() { factory.SetStorage(original) })
+}
+
+var chatImageFileDict = map[string]interface{}{
+ "id": "img-1",
+ "name": "photo.jpeg",
+ "mime_type": "image/jpeg",
+ "created_by": "user-1",
+}
+
+// Regression (no-KB chat path): the chat UI uploads attachments as file
+// dicts; AsyncChatSolo must resolve them to base64 data URIs for
+// vision-capable models. The previous implementation returned no images
+// for file dicts, so the attached image never reached the model.
+func TestSplitChatAttachments_FileDictModeReturnsDataURIs(t *testing.T) {
+ seedChatImageStorage(t)
+
+ svc := NewChatPipelineService()
+ texts, images := svc.splitChatAttachments(t.Context(), "user-1", []interface{}{chatImageFileDict})
+ if len(images) != 1 {
+ t.Fatalf("expected 1 image, got %v", images)
+ }
+ want := "data:image/jpeg;base64," + base64.StdEncoding.EncodeToString([]byte("jpeg-bytes"))
+ if images[0] != want {
+ t.Fatalf("image = %q, want data URI %q", images[0], want)
+ }
+ if texts != "" {
+ t.Fatalf("expected no text attachments for an image-only upload, got %q", texts)
+ }
+}
+
+// String-mode files keep only data:-prefixed entries as images; every
+// other non-empty entry stays in the text attachments.
+func TestSplitChatAttachments_StringModeKeepsDataURIsOnly(t *testing.T) {
+ svc := NewChatPipelineService()
+ texts, images := svc.splitChatAttachments(t.Context(), "user-1", []interface{}{
+ "data:image/png;base64,AAA", "https://example.com/a.png", "plain text", " ",
+ })
+ if len(images) != 1 || images[0] != "data:image/png;base64,AAA" {
+ t.Fatalf("unexpected images: %v", images)
+ }
+ if texts != "https://example.com/a.png\n\nplain text" {
+ t.Fatalf("unexpected text attachments: %q", texts)
+ }
+}
+
+// countingStorage wraps a Storage and counts Get calls, pinning the
+// single-fetch contract of splitChatAttachments in file-dict mode.
+type countingStorage struct {
+ storage.Storage
+ gets int
+}
+
+func (c *countingStorage) Get(ctx context.Context, bucket, fnm string, tenantID ...string) ([]byte, error) {
+ c.gets++
+ return c.Storage.Get(ctx, bucket, fnm, tenantID...)
+}
+
+// Resolving file-dict attachments must read every blob from storage exactly
+// once: the text/image split happens on a single fetch, so a vision-model
+// chat does not double-read attachment blobs.
+func TestSplitChatAttachments_FetchesStorageOnce(t *testing.T) {
+ memory := storage.NewMemoryStorage()
+ for _, id := range []string{"img-1", "img-2"} {
+ if err := memory.Put(t.Context(), "user-1-downloads", id, []byte("jpeg-bytes")); err != nil {
+ t.Fatalf("put %s: %v", id, err)
+ }
+ }
+ counter := &countingStorage{Storage: memory}
+ factory := storage.GetStorageFactory()
+ original := factory.GetStorage()
+ factory.SetStorage(counter)
+ t.Cleanup(func() { factory.SetStorage(original) })
+
+ svc := NewChatPipelineService()
+ texts, images := svc.splitChatAttachments(t.Context(), "user-1", []interface{}{
+ chatImageFileDict,
+ map[string]interface{}{"id": "img-2", "name": "photo2.jpeg", "mime_type": "image/jpeg", "created_by": "user-1"},
+ })
+ if len(images) != 2 {
+ t.Fatalf("expected 2 images, got %v", images)
+ }
+ if texts != "" {
+ t.Fatalf("expected no text attachments, got %q", texts)
+ }
+ if counter.gets != 2 {
+ t.Fatalf("storage Get called %d times for 2 files, want exactly 2", counter.gets)
+ }
+}
+
+// Regression (KB chat path): splitFileAttachments(raw=true) images feed
+// ConvertLastUserMsgToMultimodal. Raw binary blobs used to fail
+// parseDataURIOrB64, so the conversion errored and the caller silently
+// skipped it, dropping the image before the LLM call.
+func TestSplitFileAttachmentsRawImagesFeedMultimodalConversion(t *testing.T) {
+ seedChatImageStorage(t)
+
+ _, images := splitFileAttachments(t.Context(), "user-1", []interface{}{chatImageFileDict}, true)
+ if len(images) != 1 {
+ t.Fatalf("expected 1 image from splitFileAttachments, got %v", images)
+ }
+
+ converted, err := common.ConvertLastUserMsgToMultimodal(
+ map[string]interface{}{"role": "user", "content": "describe the image"},
+ images,
+ "tongyi-qianwen",
+ )
+ if err != nil {
+ t.Fatalf("ConvertLastUserMsgToMultimodal failed: %v", err)
+ }
+ rendered, ok := converted["content"].([]map[string]interface{})
+ if !ok {
+ t.Fatalf("content not rendered as parts: %T", converted["content"])
+ }
+ hasImage := false
+ for _, part := range rendered {
+ if partType, _ := part["type"].(string); partType == "image_url" {
+ hasImage = true
+ url, ok := part["image_url"].(*common.ImageURL)
+ if !ok {
+ t.Fatalf("image_url part has unexpected shape: %T", part["image_url"])
+ }
+ if !strings.HasPrefix(url.URL, "data:image/jpeg;base64,") {
+ t.Fatalf("image_url not a jpeg data URI: %q", url.URL)
+ }
+ }
+ }
+ if !hasImage {
+ t.Fatalf("no image part in converted content: %v", rendered)
+ }
+}
diff --git a/internal/service/file/file_content.go b/internal/service/file/file_content.go
index 55c86d4d5b..e7cd72a19d 100644
--- a/internal/service/file/file_content.go
+++ b/internal/service/file/file_content.go
@@ -94,7 +94,9 @@ func (s *FileService) DownloadAgentFile(ctx context.Context, tenantID, location
}
// GetFileContents fetches file contents (text + image) from storage
-// for the given file dicts.
+// for the given file dicts. Images are always returned as MIME-preserving
+// base64 data URIs so the multimodal conversion layer (parseDataURIOrB64)
+// accepts them.
//
// File dicts are the descriptors returned by the upload_info endpoint
// (UploadInfos / storeUploadInfoBlob). They contain:
@@ -107,10 +109,7 @@ func (s *FileService) DownloadAgentFile(ctx context.Context, tenantID, location
// Blobs are stored directly in "{created_by}-downloads/{id}" in object
// storage WITHOUT a corresponding File entity row in the database.
// Mirrors Python's FileService.get_files → get_blob(user_id, file_id).
-//
-// - raw=false: images returned as base64 data URIs in images; non-images parsed and returned as text.
-// - raw=true: images returned as raw bytes in images; non-images parsed and returned as text.
-func (s *FileService) GetFileContents(ctx context.Context, uid string, fileDicts []map[string]interface{}, raw bool) (texts []string, images []string, err error) {
+func (s *FileService) GetFileContents(ctx context.Context, uid string, fileDicts []map[string]interface{}) (texts []string, images []string, err error) {
storageImpl := storage.GetStorageFactory().GetStorage()
if storageImpl == nil {
return nil, nil, fmt.Errorf("storage not initialized")
@@ -139,16 +138,12 @@ func (s *FileService) GetFileContents(ctx context.Context, uid string, fileDicts
ft := utility.FilenameType(name)
if ft == utility.FileTypeVISUAL {
- if raw {
- images = append(images, string(data))
- } else {
- mediaType := strings.ToLower(strings.TrimSpace(strings.Split(mimeType, ";")[0]))
- if mediaType == "" {
- ext := utility.GetFileExtension(name)
- mediaType = utility.GetContentType(ext, string(ft))
- }
- images = append(images, "data:"+mediaType+";base64,"+base64.StdEncoding.EncodeToString(data))
+ mediaType := strings.ToLower(strings.TrimSpace(strings.Split(mimeType, ";")[0]))
+ if mediaType == "" {
+ ext := utility.GetFileExtension(name)
+ mediaType = utility.GetContentType(ext, string(ft))
}
+ images = append(images, "data:"+mediaType+";base64,"+base64.StdEncoding.EncodeToString(data))
} else {
texts = append(texts, parseFileContent(ctx, name, data))
}
diff --git a/internal/service/file/file_test.go b/internal/service/file/file_test.go
index 055354f21c..3bd4239252 100644
--- a/internal/service/file/file_test.go
+++ b/internal/service/file/file_test.go
@@ -3,7 +3,7 @@ package file
import (
"bytes"
"context"
-
+ "encoding/base64"
"errors"
"io"
"net/http"
@@ -149,7 +149,7 @@ func TestFileService_GetFileContents_NotAccessible(t *testing.T) {
"name": "secret.txt",
"mime_type": "text/plain",
"created_by": "other-user",
- }}, false)
+ }})
if err == nil {
t.Fatal("expected authorization error")
}
@@ -178,7 +178,7 @@ func TestFileService_GetFileContents_Accessible(t *testing.T) {
"name": "doc.txt",
"mime_type": "text/plain",
"created_by": "user-1",
- }}, false)
+ }})
if err != nil {
t.Fatalf("GetFileContents failed: %v", err)
}
@@ -190,6 +190,42 @@ func TestFileService_GetFileContents_Accessible(t *testing.T) {
}
}
+// Regression for chat image attachments: visual file dicts must come back
+// as mime-preserving base64 data URIs. The Go multimodal conversion layer
+// (common.ConvertLastUserMsgToMultimodal → parseDataURIOrB64) rejects raw
+// binary blobs, which silently dropped the image before the LLM call.
+func TestFileService_GetFileContents_ImageAttachmentAsDataURI(t *testing.T) {
+ memory := storage.NewMemoryStorage()
+ if err := memory.Put(t.Context(), "user-1-downloads", "img-1", []byte("jpeg-bytes")); err != nil {
+ t.Fatalf("put: %v", err)
+ }
+ factory := storage.GetStorageFactory()
+ originalStorage := factory.GetStorage()
+ factory.SetStorage(memory)
+ t.Cleanup(func() { factory.SetStorage(originalStorage) })
+
+ svc := testFileService()
+ texts, images, err := svc.GetFileContents(t.Context(), "user-1", []map[string]interface{}{{
+ "id": "img-1",
+ "name": "25336bcdd3126d32a522db5e9c9fcaf3.jpeg",
+ "mime_type": "image/jpeg",
+ "created_by": "user-1",
+ }})
+ if err != nil {
+ t.Fatalf("GetFileContents failed: %v", err)
+ }
+ if len(texts) != 0 {
+ t.Fatalf("expected no texts for image attachment, got %v", texts)
+ }
+ if len(images) != 1 {
+ t.Fatalf("expected 1 image, got %v", images)
+ }
+ want := "data:image/jpeg;base64," + base64.StdEncoding.EncodeToString([]byte("jpeg-bytes"))
+ if images[0] != want {
+ t.Fatalf("image = %q, want data URI %q", images[0], want)
+ }
+}
+
func TestFileService_ParseAgentUploads_TextAndImageInRequestOrder(t *testing.T) {
ctx := t.Context()
memory := storage.NewMemoryStorage()