RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(chat): deliver uploaded image attachments to vision models in Go chat pipeline (#18708)
euvre874d156c5734
diff --git a/internal/service/chat_pipeline.go b/internal/service/chat_pipeline.goindex 2dde6f6cb9..bca2dfb4d8 100644--- a/internal/service/chat_pipeline.go+++ b/internal/service/chat_pipeline.go@@ -1367,13 +1367,18 @@ func (s *ChatPipelineService) AsyncChatSolo(isImage2Text := modelType == "image2text"if len(messages) > 0 {if files, hasFiles := messages[len(messages)-1]["files"]; hasFiles {- attachmentsStr = s.processFileAttachments(ctx, userID, files)+ var images []string+ attachmentsStr, images = s.splitChatAttachments(ctx, userID, files)if isImage2Text {- imageFiles = s.extractRawImageURLs(files)+ imageFiles = images} else {common.Debug("AsyncChatSolo: dropping image attachments for text-only chat model",zap.String("llm_id", chat.LLMID))}+ common.Info("AsyncChatSolo: file attachments resolved",+ zap.Bool("vision_model", isImage2Text),+ zap.Int("image_files", len(imageFiles)),+ zap.Int("text_attachment_bytes", len(attachmentsStr)))}}@@ -1623,53 +1628,71 @@ func (s *ChatPipelineService) AsyncChatSolo(return out, nil}-// extractRawImageURLs extracts image references as raw URLs/data-URIs from-// the string-mode files list, WITHOUT fetching blobs and WITHOUT filtering-// to data: prefixes. Used for image2text models that expect URLs in the-// multimodal content (matches Python's `image_files` from-// `split_file_attachments(files, raw=True)` at-// dialog_service.py:371-392).+// splitChatAttachments resolves the last message's file attachments into+// text content (entries joined by "\n\n") and image data URIs, reading+// each blob from storage at most once.//-// The downstream ConvertLastUserMsgToMultimodal calls parseDataURIOrB64-// (multimodal.go:63-92) which correctly handles all three forms:-// - data: URI → base64 source-// - http:// or https:// URL → URL source-// - raw base64 → base64 source (default media type)+// - File-dict mode (the chat UI's upload_info flow): FileService.+// GetFileContents fetches the blobs; non-visual files are parsed to+// text, visual files come back as base64 data URIs.+// - String mode (pre-resolved content): data:-prefixed entries become+// images; the remaining non-empty entries become text.//-// File-dict mode is a known limitation: returns empty for now. A future-// FileService.GetFileURLsForChat (mirror of GetFileContents with-// raw=true) would be needed to fully cover the file-dict + image2text-// combination. The Python equivalent has the same limitation-// (split_file_attachments calls FileService.get_files which doesn't-// fetch blobs in raw mode).-func (s *ChatPipelineService) extractRawImageURLs(files interface{}) []string {+// Downstream ConvertLastUserMsgToMultimodal → parseDataURIOrB64 accepts the+// returned data URIs directly.+func (s *ChatPipelineService) splitChatAttachments(ctx context.Context, userID string, files interface{}) (string, []string) {+ // ── File-dict mode ──if fileDicts, ok := parseFileDicts(files); ok {- _ = fileDicts // see file-dict limitation comment above- common.Debug("AsyncChatSolo: file-dict + image2text not yet supported; image refs dropped",- zap.Int("file_dict_count", len(fileDicts)))- return nil+ // Only used for GetFileContents (read-only); nil DocRemover means+ // this FileService MUST NOT be used for DeleteFiles.+ fileSvc := file.NewFileService(CheckFileTeamPermission, nil)+ texts, images, err := fileSvc.GetFileContents(ctx, userID, fileDicts)+ if err != nil {+ common.Warn("GetFileContents failed in splitChatAttachments",+ zap.Error(err))+ return "", nil+ }+ if len(texts) == 0 {+ return "", images+ }+ return strings.Join(texts, "\n\n"), images}- // String-mode: return all entries as-is. The downstream- // ConvertLastUserMsgToMultimodal + parseDataURIOrB64 will- // dispatch on prefix (data: → base64, http(s): → url, else →- // raw base64).- var urls []string+ // ── String mode ──+ var texts []string+ var images []stringswitch v := files.(type) {case []string:for _, f := range v {- if f != "" {- urls = append(urls, f)+ if f = strings.TrimSpace(f); f == "" {+ continue+ }+ if strings.HasPrefix(f, "data:") {+ images = append(images, f)+ } else {+ texts = append(texts, f)}}case []interface{}:for _, f := range v {- if s, ok := f.(string); ok && s != "" {- urls = append(urls, s)+ str, ok := f.(string)+ if !ok {+ continue+ }+ if str = strings.TrimSpace(str); str == "" {+ continue+ }+ if strings.HasPrefix(str, "data:") {+ images = append(images, str)+ } else {+ texts = append(texts, str)}}}- return urls+ if len(texts) == 0 {+ return "", images+ }+ return strings.Join(texts, "\n\n"), images}// ---------------------------------------------------------------------------@@ -2070,53 +2093,6 @@ func lastUserQuestion(messages []map[string]interface{}) string {return ""}-// processFileAttachments extracts text content from file attachments.-// Mirrors Python's split_file_attachments (dialog_service.py:371-392)-// in raw=false mode: returns text attachments joined by "\n\n",-// filtering out data-URI image attachments.-//-// When files are file dicts (Python-compatible format), calls-// FileService.GetFileContents to fetch actual blobs from storage.-func (s *ChatPipelineService) processFileAttachments(ctx context.Context, userID string, files interface{}) string {- // ── File-dict mode ──- if fileDicts, ok := parseFileDicts(files); ok {- // Only used for GetFileContents (read-only); nil DocRemover means- // this FileService MUST NOT be used for DeleteFiles.- fileSvc := file.NewFileService(CheckFileTeamPermission, nil)- texts, _, err := fileSvc.GetFileContents(ctx, userID, fileDicts, false)- if err != nil {- common.Warn("GetFileContents failed in processFileAttachments",- zap.Error(err))- return ""- }- if len(texts) == 0 {- return ""- }- return strings.Join(texts, "\n\n")- }-- // ── String fallback ──- var texts []string- switch v := files.(type) {- case []string:- for _, f := range v {- if s := strings.TrimSpace(f); s != "" && !strings.HasPrefix(s, "data:") {- texts = append(texts, s)- }- }- case []interface{}:- for _, f := range v {- if s, ok := f.(string); ok && strings.TrimSpace(s) != "" && !strings.HasPrefix(s, "data:") {- texts = append(texts, s)- }- }- }- if len(texts) == 0 {- return ""- }- return strings.Join(texts, "\n\n")-}-// splitFileAttachments mirrors Python's `split_file_attachments` at// dialog_service.py:371-392. It separates `messages[-1]["files"]`// into text-file content and image attachments.@@ -2125,8 +2101,8 @@ func (s *ChatPipelineService) processFileAttachments(ctx context.Context, userID//// 1. File-dict mode: When `files` is `[]map[string]interface{}` (each dict// with keys "id", "created_by", "mime_type", "name"), the method calls-// FileService.GetFileContents to fetch actual file blobs from-// storage, mirroring Python's FileService.get_files().+// FileService.GetFileContents to fetch actual file blobs from storage;+// images come back as base64 data URIs.//// 2. String-fallback mode: When `files` is `[]string` or `[]interface{}` of// strings (pre-resolved content), the method does simple string splitting:@@ -2140,7 +2116,7 @@ func splitFileAttachments(ctx context.Context, userID string, files interface{},// Only used for GetFileContents (read-only); nil DocRemover means// this FileService MUST NOT be used for DeleteFiles.fileSvc := file.NewFileService(CheckFileTeamPermission, nil)- texts, images, err := fileSvc.GetFileContents(ctx, userID, fileDicts, raw)+ texts, images, err := fileSvc.GetFileContents(ctx, userID, fileDicts)if err != nil {common.Warn("GetFileContents failed, falling back to string splitting",zap.Error(err))diff --git a/internal/service/chat_pipeline_image_test.go b/internal/service/chat_pipeline_image_test.gonew file mode 100644index 0000000000..6aa7d479b7--- /dev/null+++ b/internal/service/chat_pipeline_image_test.go@@ -0,0 +1,170 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package service++import (+ "context"+ "encoding/base64"+ "strings"+ "testing"++ "ragflow/internal/common"+ "ragflow/internal/storage"+)++// seedChatImageStorage installs an in-memory storage holding one jpeg blob+// under user-1's downloads bucket, the layout the upload_info flow writes.+func seedChatImageStorage(t *testing.T) {+ t.Helper()+ memory := storage.NewMemoryStorage()+ if err := memory.Put(t.Context(), "user-1-downloads", "img-1", []byte("jpeg-bytes")); err != nil {+ t.Fatalf("put: %v", err)+ }+ factory := storage.GetStorageFactory()+ original := factory.GetStorage()+ factory.SetStorage(memory)+ t.Cleanup(func() { factory.SetStorage(original) })+}++var chatImageFileDict = map[string]interface{}{+ "id": "img-1",+ "name": "photo.jpeg",+ "mime_type": "image/jpeg",+ "created_by": "user-1",+}++// Regression (no-KB chat path): the chat UI uploads attachments as file+// dicts; AsyncChatSolo must resolve them to base64 data URIs for+// vision-capable models. The previous implementation returned no images+// for file dicts, so the attached image never reached the model.+func TestSplitChatAttachments_FileDictModeReturnsDataURIs(t *testing.T) {+ seedChatImageStorage(t)++ svc := NewChatPipelineService()+ texts, images := svc.splitChatAttachments(t.Context(), "user-1", []interface{}{chatImageFileDict})+ if len(images) != 1 {+ t.Fatalf("expected 1 image, got %v", images)+ }+ want := "data:image/jpeg;base64," + base64.StdEncoding.EncodeToString([]byte("jpeg-bytes"))+ if images[0] != want {+ t.Fatalf("image = %q, want data URI %q", images[0], want)+ }+ if texts != "" {+ t.Fatalf("expected no text attachments for an image-only upload, got %q", texts)+ }+}++// String-mode files keep only data:-prefixed entries as images; every+// other non-empty entry stays in the text attachments.+func TestSplitChatAttachments_StringModeKeepsDataURIsOnly(t *testing.T) {+ svc := NewChatPipelineService()+ texts, images := svc.splitChatAttachments(t.Context(), "user-1", []interface{}{+ "data:image/png;base64,AAA", "https://example.com/a.png", "plain text", " ",+ })+ if len(images) != 1 || images[0] != "data:image/png;base64,AAA" {+ t.Fatalf("unexpected images: %v", images)+ }+ if texts != "https://example.com/a.png\n\nplain text" {+ t.Fatalf("unexpected text attachments: %q", texts)+ }+}++// countingStorage wraps a Storage and counts Get calls, pinning the+// single-fetch contract of splitChatAttachments in file-dict mode.+type countingStorage struct {+ storage.Storage+ gets int+}++func (c *countingStorage) Get(ctx context.Context, bucket, fnm string, tenantID ...string) ([]byte, error) {+ c.gets+++ return c.Storage.Get(ctx, bucket, fnm, tenantID...)+}++// Resolving file-dict attachments must read every blob from storage exactly+// once: the text/image split happens on a single fetch, so a vision-model+// chat does not double-read attachment blobs.+func TestSplitChatAttachments_FetchesStorageOnce(t *testing.T) {+ memory := storage.NewMemoryStorage()+ for _, id := range []string{"img-1", "img-2"} {+ if err := memory.Put(t.Context(), "user-1-downloads", id, []byte("jpeg-bytes")); err != nil {+ t.Fatalf("put %s: %v", id, err)+ }+ }+ counter := &countingStorage{Storage: memory}+ factory := storage.GetStorageFactory()+ original := factory.GetStorage()+ factory.SetStorage(counter)+ t.Cleanup(func() { factory.SetStorage(original) })++ svc := NewChatPipelineService()+ texts, images := svc.splitChatAttachments(t.Context(), "user-1", []interface{}{+ chatImageFileDict,+ map[string]interface{}{"id": "img-2", "name": "photo2.jpeg", "mime_type": "image/jpeg", "created_by": "user-1"},+ })+ if len(images) != 2 {+ t.Fatalf("expected 2 images, got %v", images)+ }+ if texts != "" {+ t.Fatalf("expected no text attachments, got %q", texts)+ }+ if counter.gets != 2 {+ t.Fatalf("storage Get called %d times for 2 files, want exactly 2", counter.gets)+ }+}++// Regression (KB chat path): splitFileAttachments(raw=true) images feed+// ConvertLastUserMsgToMultimodal. Raw binary blobs used to fail+// parseDataURIOrB64, so the conversion errored and the caller silently+// skipped it, dropping the image before the LLM call.+func TestSplitFileAttachmentsRawImagesFeedMultimodalConversion(t *testing.T) {+ seedChatImageStorage(t)++ _, images := splitFileAttachments(t.Context(), "user-1", []interface{}{chatImageFileDict}, true)+ if len(images) != 1 {+ t.Fatalf("expected 1 image from splitFileAttachments, got %v", images)+ }++ converted, err := common.ConvertLastUserMsgToMultimodal(+ map[string]interface{}{"role": "user", "content": "describe the image"},+ images,+ "tongyi-qianwen",+ )+ if err != nil {+ t.Fatalf("ConvertLastUserMsgToMultimodal failed: %v", err)+ }+ rendered, ok := converted["content"].([]map[string]interface{})+ if !ok {+ t.Fatalf("content not rendered as parts: %T", converted["content"])+ }+ hasImage := false+ for _, part := range rendered {+ if partType, _ := part["type"].(string); partType == "image_url" {+ hasImage = true+ url, ok := part["image_url"].(*common.ImageURL)+ if !ok {+ t.Fatalf("image_url part has unexpected shape: %T", part["image_url"])+ }+ if !strings.HasPrefix(url.URL, "data:image/jpeg;base64,") {+ t.Fatalf("image_url not a jpeg data URI: %q", url.URL)+ }+ }+ }+ if !hasImage {+ t.Fatalf("no image part in converted content: %v", rendered)+ }+}diff --git a/internal/service/file/file_content.go b/internal/service/file/file_content.goindex 55c86d4d5b..e7cd72a19d 100644--- a/internal/service/file/file_content.go+++ b/internal/service/file/file_content.go@@ -94,7 +94,9 @@ func (s *FileService) DownloadAgentFile(ctx context.Context, tenantID, location}// GetFileContents fetches file contents (text + image) from storage-// for the given file dicts.+// for the given file dicts. Images are always returned as MIME-preserving+// base64 data URIs so the multimodal conversion layer (parseDataURIOrB64)+// accepts them.//// File dicts are the descriptors returned by the upload_info endpoint// (UploadInfos / storeUploadInfoBlob). They contain:@@ -107,10 +109,7 @@ func (s *FileService) DownloadAgentFile(ctx context.Context, tenantID, location// Blobs are stored directly in "{created_by}-downloads/{id}" in object// storage WITHOUT a corresponding File entity row in the database.// Mirrors Python's FileService.get_files → get_blob(user_id, file_id).-//-// - raw=false: images returned as base64 data URIs in images; non-images parsed and returned as text.-// - raw=true: images returned as raw bytes in images; non-images parsed and returned as text.-func (s *FileService) GetFileContents(ctx context.Context, uid string, fileDicts []map[string]interface{}, raw bool) (texts []string, images []string, err error) {+func (s *FileService) GetFileContents(ctx context.Context, uid string, fileDicts []map[string]interface{}) (texts []string, images []string, err error) {storageImpl := storage.GetStorageFactory().GetStorage()if storageImpl == nil {return nil, nil, fmt.Errorf("storage not initialized")@@ -139,16 +138,12 @@ func (s *FileService) GetFileContents(ctx context.Context, uid string, fileDictsft := utility.FilenameType(name)if ft == utility.FileTypeVISUAL {- if raw {- images = append(images, string(data))- } else {- mediaType := strings.ToLower(strings.TrimSpace(strings.Split(mimeType, ";")[0]))- if mediaType == "" {- ext := utility.GetFileExtension(name)- mediaType = utility.GetContentType(ext, string(ft))- }- images = append(images, "data:"+mediaType+";base64,"+base64.StdEncoding.EncodeToString(data))+ mediaType := strings.ToLower(strings.TrimSpace(strings.Split(mimeType, ";")[0]))+ if mediaType == "" {+ ext := utility.GetFileExtension(name)+ mediaType = utility.GetContentType(ext, string(ft))}+ images = append(images, "data:"+mediaType+";base64,"+base64.StdEncoding.EncodeToString(data))} else {texts = append(texts, parseFileContent(ctx, name, data))}diff --git a/internal/service/file/file_test.go b/internal/service/file/file_test.goindex 055354f21c..3bd4239252 100644--- a/internal/service/file/file_test.go+++ b/internal/service/file/file_test.go@@ -3,7 +3,7 @@ package fileimport ("bytes""context"-+ "encoding/base64""errors""io""net/http"@@ -149,7 +149,7 @@ func TestFileService_GetFileContents_NotAccessible(t *testing.T) {"name": "secret.txt","mime_type": "text/plain","created_by": "other-user",- }}, false)+ }})if err == nil {t.Fatal("expected authorization error")}@@ -178,7 +178,7 @@ func TestFileService_GetFileContents_Accessible(t *testing.T) {"name": "doc.txt","mime_type": "text/plain","created_by": "user-1",- }}, false)+ }})if err != nil {t.Fatalf("GetFileContents failed: %v", err)}@@ -190,6 +190,42 @@ func TestFileService_GetFileContents_Accessible(t *testing.T) {}}+// Regression for chat image attachments: visual file dicts must come back+// as mime-preserving base64 data URIs. The Go multimodal conversion layer+// (common.ConvertLastUserMsgToMultimodal → parseDataURIOrB64) rejects raw+// binary blobs, which silently dropped the image before the LLM call.+func TestFileService_GetFileContents_ImageAttachmentAsDataURI(t *testing.T) {+ memory := storage.NewMemoryStorage()+ if err := memory.Put(t.Context(), "user-1-downloads", "img-1", []byte("jpeg-bytes")); err != nil {+ t.Fatalf("put: %v", err)+ }+ factory := storage.GetStorageFactory()+ originalStorage := factory.GetStorage()+ factory.SetStorage(memory)+ t.Cleanup(func() { factory.SetStorage(originalStorage) })++ svc := testFileService()+ texts, images, err := svc.GetFileContents(t.Context(), "user-1", []map[string]interface{}{{+ "id": "img-1",+ "name": "25336bcdd3126d32a522db5e9c9fcaf3.jpeg",+ "mime_type": "image/jpeg",+ "created_by": "user-1",+ }})+ if err != nil {+ t.Fatalf("GetFileContents failed: %v", err)+ }+ if len(texts) != 0 {+ t.Fatalf("expected no texts for image attachment, got %v", texts)+ }+ if len(images) != 1 {+ t.Fatalf("expected 1 image, got %v", images)+ }+ want := "data:image/jpeg;base64," + base64.StdEncoding.EncodeToString([]byte("jpeg-bytes"))+ if images[0] != want {+ t.Fatalf("image = %q, want data URI %q", images[0], want)+ }+}+func TestFileService_ParseAgentUploads_TextAndImageInRequestOrder(t *testing.T) {ctx := t.Context()memory := storage.NewMemoryStorage()