RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(chat): preserve images across reasoning and conversation history (#20728)
Sbaaoui Idrissf6477b197768
diff --git a/conf/models/bedrock.json b/conf/models/bedrock.jsonindex b322175c44..b24e1aa7bd 100644--- a/conf/models/bedrock.json+++ b/conf/models/bedrock.json@@ -12,7 +12,8 @@"context_length": 200000,"max_output": 8192,"model_types": [- "chat"+ "chat",+ "vision"]},{@@ -28,7 +29,8 @@"context_length": 200000,"max_output": 4096,"model_types": [- "chat"+ "chat",+ "vision"]},{@@ -36,7 +38,8 @@"context_length": 200000,"max_output": 4096,"model_types": [- "chat"+ "chat",+ "vision"]},{@@ -44,7 +47,8 @@"context_length": 200000,"max_output": 4096,"model_types": [- "chat"+ "chat",+ "vision"]},{@@ -92,7 +96,8 @@"context_length": 300000,"max_output": 5000,"model_types": [- "chat"+ "chat",+ "vision"]},{@@ -100,7 +105,8 @@"context_length": 300000,"max_output": 5000,"model_types": [- "chat"+ "chat",+ "vision"]},{diff --git a/conf/models/cohere.json b/conf/models/cohere.jsonindex e0c37bc01f..24fb86854c 100644--- a/conf/models/cohere.json+++ b/conf/models/cohere.json@@ -18,7 +18,8 @@"context_length": 131072,"max_output": 65536,"model_types": [- "chat"+ "chat",+ "vision"]},{@@ -58,7 +59,8 @@"context_length": 131072,"max_output": 8192,"model_types": [- "chat"+ "chat",+ "vision"]},{diff --git a/conf/models/minimax.json b/conf/models/minimax.jsonindex cdd3ede5d3..59fd934680 100644--- a/conf/models/minimax.json+++ b/conf/models/minimax.json@@ -18,7 +18,8 @@"context_length": 1024000,"max_output": 131072,"model_types": [- "chat"+ "chat",+ "vision"],"thinking": {"default_value": true,diff --git a/conf/models/mistral.json b/conf/models/mistral.jsonindex 11bd30eb16..2119046046 100644--- a/conf/models/mistral.json+++ b/conf/models/mistral.json@@ -18,7 +18,8 @@"context_length": 262144,"max_output": 262144,"model_types": [- "chat"+ "chat",+ "vision"],"tools": {"support": true@@ -41,7 +42,8 @@"context_length": 262144,"max_output": 262144,"model_types": [- "chat"+ "chat",+ "vision"],"tools": {"support": true@@ -52,7 +54,8 @@"context_length": 262144,"max_output": 262144,"model_types": [- "chat"+ "chat",+ "vision"],"tools": {"support": true@@ -63,7 +66,8 @@"context_length": 262144,"max_output": 262144,"model_types": [- "chat"+ "chat",+ "vision"],"tools": {"support": true@@ -74,7 +78,8 @@"context_length": 262144,"max_output": 262144,"model_types": [- "chat"+ "chat",+ "vision"],"tools": {"support": truediff --git a/conf/models/novita.json b/conf/models/novita.jsonindex ae357c8784..234e01c3fa 100644--- a/conf/models/novita.json+++ b/conf/models/novita.json@@ -72,7 +72,8 @@"context_length": 98304,"max_output": 16384,"model_types": [- "chat"+ "chat",+ "vision"],"tools": {"support": truediff --git a/conf/models/openai.json b/conf/models/openai.jsonindex 9f2cd70350..e0973fa050 100644--- a/conf/models/openai.json+++ b/conf/models/openai.json@@ -357,7 +357,8 @@"context_length": 128000,"max_output": 4096,"model_types": [- "chat"+ "chat",+ "vision"],"tools": {"support": truediff --git a/conf/models/siliconflow.json b/conf/models/siliconflow.jsonindex 58caa25a81..87399b1a9e 100644--- a/conf/models/siliconflow.json+++ b/conf/models/siliconflow.json@@ -917,7 +917,8 @@"context_length": 262144,"max_output": 262144,"model_types": [- "chat"+ "chat",+ "vision"],"tools": {"support": truediff --git a/internal/agentic_rag/agent.go b/internal/agentic_rag/agent.goindex 332393313a..86a38a9cca 100644--- a/internal/agentic_rag/agent.go+++ b/internal/agentic_rag/agent.go@@ -390,9 +390,10 @@ func Run(ctx context.Context, in Input) (string, error) {// EnableStreaming lives on RunnerConfig, not ChatModelAgentConfig.explorerHead := sess.explorer.head(ctx)- question := lastUserQuestion(in.Messages)+ question := schema.UserMessage(lastUserQuestion(in.Messages))seed := in.Messagesif len(seed) > 0 {+ question = seed[len(seed)-1]// The final user message is the new turn and must be appended exactly// once by turnMessages.seed = seed[:len(seed)-1]@@ -721,10 +722,12 @@ func finalizeAnswer("assumption explicitly, and still provide the computed answer. Do not request any tools " +"and do not hedge with \"I would need\" or \"I could not find\".")- resp, err := m.Generate(ctx, []*schema.Message{{+ finalMessages := append([]*schema.Message(nil), messages...)+ finalMessages = append(finalMessages, &schema.Message{Role: schema.User,Content: body.String(),- }})+ })+ resp, err := m.Generate(ctx, finalMessages)if err != nil {return "", err}diff --git a/internal/agentic_rag/run_session.go b/internal/agentic_rag/run_session.goindex a4fc1d8d0a..cff1d5c1f3 100644--- a/internal/agentic_rag/run_session.go+++ b/internal/agentic_rag/run_session.go@@ -189,9 +189,9 @@ func (c *conversation) markSeeded() {// deliverable first — a repair turn that lost its predecessor's events to a// restart must still see the question it is answering — every later turn sends// the directive alone and lets the Runner replay the rest.-func (c *conversation) turnMessages(seed []*schema.Message, final, directive string) []adk.Message {+func (c *conversation) turnMessages(seed []*schema.Message, final string, directive *schema.Message) []adk.Message {if c.seeded {- return []adk.Message{schema.UserMessage(directive)}+ return []adk.Message{directive}}c.markSeeded()msgs := make([]adk.Message, 0, len(seed)+2)@@ -201,7 +201,7 @@ func (c *conversation) turnMessages(seed []*schema.Message, final, directive str// standing deliverable the directive alone carries the ask.msgs = append(msgs, schema.AssistantMessage(final, nil))}- return append(msgs, schema.UserMessage(directive))+ return append(msgs, directive)}// lastAssistant walks the conversation's message events NEWEST FIRST anddiff --git a/internal/agentic_rag/run_session_test.go b/internal/agentic_rag/run_session_test.goindex 436ce16c20..5fe1c1479f 100644--- a/internal/agentic_rag/run_session_test.go+++ b/internal/agentic_rag/run_session_test.go@@ -139,7 +139,7 @@ func TestConversationRestartWhenNothingCommitted(t *testing.T) {t.Error("a restarted conversation must re-seed the caller's history")}base := []*schema.Message{schema.UserMessage("who signed?")}- got := conv.turnMessages(base, "Final Answer: **1897**", "FIX IT")+ got := conv.turnMessages(base, "Final Answer: **1897**", schema.UserMessage("FIX IT"))if len(got) != 3 || got[0].Content != "who signed?" || got[2].Content != "FIX IT" {t.Fatalf("reseeded turn = %v, want history + final + directive", got)}@@ -153,15 +153,29 @@ func TestConversationRestartWhenNothingCommitted(t *testing.T) {// The caller's chat history belongs to ONE turn: later turns hand over nothing// but their own message.+func TestConversationTurnMessagesPreservesImages(t *testing.T) {+ conv := newTestConversation()+ uri := "data:image/jpeg;base64,AAAA"+ question := schema.UserMessage("describe")+ question.UserInputMultiContent = []schema.MessageInputPart{{+ Type: schema.ChatMessagePartTypeImageURL,+ Image: &schema.MessageInputImage{MessagePartCommon: schema.MessagePartCommon{URL: &uri}},+ }}+ got := conv.turnMessages(nil, "", question)+ if len(got) != 1 || len(got[0].UserInputMultiContent) != 1 {+ t.Fatalf("got %#v, want image-bearing user turn", got)+ }+}+func TestConversationTurnMessagesSeedsOnce(t *testing.T) {conv := newTestConversation()base := []*schema.Message{schema.UserMessage("who signed?")}- first := conv.turnMessages(base, "", "FIX IT")+ first := conv.turnMessages(base, "", schema.UserMessage("FIX IT"))if len(first) != 2 || first[0].Content != "who signed?" || first[1].Content != "FIX IT" {t.Fatalf("unseeded turn = %v, want history + directive", first)}- second := conv.turnMessages(base, "Final Answer: **1897**", "AGAIN")+ second := conv.turnMessages(base, "Final Answer: **1897**", schema.UserMessage("AGAIN"))if len(second) != 1 || second[0].Content != "AGAIN" {t.Fatalf("seeded turn = %v, want the directive alone", second)}diff --git a/internal/service/agentic_pipeline.go b/internal/service/agentic_pipeline.goindex 5507e8b961..4ae0c4394d 100644--- a/internal/service/agentic_pipeline.go+++ b/internal/service/agentic_pipeline.go@@ -114,6 +114,24 @@ func (s *ChatPipelineService) agenticRag(return}docIDs := s.chatMetadataDocIDs(runCtx, chat, lastUserQuestion(messages), chain[0], nil, chatDatasetIDs(chat))+ modelType := "chat"+ if info := chain[0].Info(); info != nil && entity.ModelTypeFromStrings(info.ModelTypes).Has(entity.ModelTypeImage2Text) {+ modelType = "image2text"+ }+ agentMessages := s.convertMessagesToEino(runCtx, userID, modelType, messages)+ // Image-bearing turns cannot fail over to a text-only model.+ for _, msg := range agentMessages {+ if len(msg.UserInputMultiContent) > 0 {+ visionChain := chain[:0]+ for _, candidate := range chain {+ if info := candidate.Info(); info != nil && entity.ModelTypeFromStrings(info.ModelTypes).Has(entity.ModelTypeImage2Text) {+ visionChain = append(visionChain, candidate)+ }+ }+ chain = visionChain+ break+ }+ }// Apply the dialog's LLM settings with per-request overrides exactly// like the regular AsyncChat path does, then let the template pin its@@ -163,7 +181,7 @@ func (s *ChatPipelineService) agenticRag(final, runErr := agentic_rag.Run(runCtx, agentic_rag.Input{Model: model,SynthModel: synth,- Messages: convertMessagesToEino(messages),+ Messages: agentMessages,TemplateID: mode,TenantID: chat.TenantID,DatasetIDs: chatDatasetIDs(chat),@@ -352,16 +370,30 @@ func (s *ChatPipelineService) agenticWebSearch(promptConfig map[string]interface}// convertMessagesToEino converts pre-filtered user/assistant messages into-// eino schema messages. Only string content is supported; multimodal parts are-// not carried into the ReAct loop.-func convertMessagesToEino(messages []map[string]interface{}) []*schema.Message {+// eino schema messages, resolving uploaded attachments throughout the history.+func (s *ChatPipelineService) convertMessagesToEino(ctx context.Context, userID, modelType string, messages []map[string]interface{}) []*schema.Message {out := make([]*schema.Message, 0, len(messages))for _, m := range messages {role, _ := m["role"].(string)content, _ := m["content"].(string)switch role {case "user":- out = append(out, schema.UserMessage(content))+ text, images := s.splitChatAttachments(ctx, userID, m["files"])+ if text != "" {+ content += "\n\n" + text+ }+ msg := schema.UserMessage(content)+ images, _ = gateImageAttachments("", modelType, images)+ if len(images) > 0 {+ msg.UserInputMultiContent = append(msg.UserInputMultiContent, schema.MessageInputPart{Type: schema.ChatMessagePartTypeText, Text: content})+ for _, uri := range images {+ msg.UserInputMultiContent = append(msg.UserInputMultiContent, schema.MessageInputPart{+ Type: schema.ChatMessagePartTypeImageURL,+ Image: &schema.MessageInputImage{MessagePartCommon: schema.MessagePartCommon{URL: new(uri)}},+ })+ }+ }+ out = append(out, msg)case "assistant":out = append(out, schema.AssistantMessage(content, nil))default:diff --git a/internal/service/chat_pipeline.go b/internal/service/chat_pipeline.goindex 5b2c9518fd..99360b4436 100644--- a/internal/service/chat_pipeline.go+++ b/internal/service/chat_pipeline.go@@ -438,6 +438,17 @@ func (s *ChatPipelineService) AsyncChat(zap.Int("attachment_bytes", len(attachments)))}+ var historyImageFiles []string+ for _, previous := range messages[:len(messages)-1] {+ if previous["role"] == "user" && previous["files"] != nil {+ _, images := s.splitChatAttachments(ctx, userID, previous["files"])+ modelType, _ := llmModelConfig["model_type"].(string)+ images, _ = gateImageAttachments(chat.LLMID, modelType, images)+ historyImageFiles = append(historyImageFiles, images...)+ }+ }+ hasImageAttachments = hasImageAttachments || len(historyImageFiles) > 0+// === Phase 6: SQL Retrieval ===// Retrieve field_map for SQL retrieval (preferred over vector search)promptConfig := chat.PromptConfig@@ -835,11 +846,12 @@ func (s *ChatPipelineService) AsyncChat(}harnessSystemPrompt = s.formatPrompt(sp, kws)}+ harnessImages := append(historyImageFiles, imageFiles...)var history []map[string]interface{}if kwargs["store_history_messages"] == false {history = messages}- hk, slotCites, citeChunkIDs, harnessAnswer, hErr := s.retrieveViaHarness(ctx, question, kbs, docIDs, imageFiles, attachments, thinkingMode, chat.TenantID, chat.LLMID, chat.ID, HarnessRetrieval{+ hk, slotCites, citeChunkIDs, harnessAnswer, hErr := s.retrieveViaHarness(ctx, question, kbs, docIDs, harnessImages, attachments, thinkingMode, chat.TenantID, chat.LLMID, chat.ID, HarnessRetrieval{TopN: int(chat.TopN),SimilarityThreshold: chat.SimilarityThreshold,VectorSimilarityWeight: chat.VectorSimilarityWeight,@@ -1134,12 +1146,16 @@ func (s *ChatPipelineService) AsyncChat(if factoryName == "" {factoryName = factoryFromLLMID(chat.LLMID)}- for _, m := range messages {+ modelType, _ := llmModelConfig["model_type"].(string)+ for i, m := range messages {role, _ := m["role"].(string)if role == "system" && kwargs["store_history_messages"] != false {continue}llmMessage := normalizeLLMMessage(m)+ if i < len(messages)-1 {+ llmMessage["files"] = m["files"]+ }content := llmMessage["content"]if contentStr, ok := content.(string); ok {content = cleanCitationMarkers(contentStr)@@ -1155,6 +1171,9 @@ func (s *ChatPipelineService) AsyncChat(// Fit messages within token budget.usedTokenCount, llmMessages := s.messageFitIn(llmMessages, int(float64(modelMaxTokens)*0.95))+ for i := range llmMessages[:len(llmMessages)-1] {+ llmMessages[i] = s.chatHistoryMessage(ctx, userID, llmMessages[i], modelType, factoryName)+ }common.Debug("Messages fitted in token budget",zap.Int("model max_tokens", modelMaxTokens),zap.Int("used_token_count", usedTokenCount),@@ -1625,11 +1644,14 @@ func (s *ChatPipelineService) AsyncChatSolo(// 3. Strip citation markers; non-storing model tests retain payload system messages.var msg []map[string]interface{}- for _, m := range messages {+ for i, m := range messages {role, _ := m["role"].(string)if role == "system" && config["store_history_messages"] != false {continue}+ if i < len(messages)-1 {+ m = s.chatHistoryMessage(ctx, userID, m, modelType, factoryName)+ }llmMessage := normalizeLLMMessage(m)content := llmMessage["content"]if contentStr, ok := content.(string); ok {@@ -2854,6 +2876,21 @@ func (s *ChatPipelineService) buildChatMessages(systemContent string, messages [return result}+func (s *ChatPipelineService) chatHistoryMessage(ctx context.Context, userID string, message map[string]interface{}, modelType, factory string) map[string]interface{} {+ if message["role"] != "user" || message["files"] == nil {+ return message+ }+ _, images := s.splitChatAttachments(ctx, userID, message["files"])+ images, _ = gateImageAttachments("", modelType, images)+ msg := normalizeLLMMessage(message)+ converted, err := common.ConvertLastUserMsgToMultimodal(msg, images, factory)+ if err != nil {+ common.Warn("chat history multimodal conversion failed", zap.Error(err))+ return msg+ }+ return converted+}+func normalizeLLMMessage(message map[string]interface{}) map[string]interface{} {normalized := make(map[string]interface{}, len(llmMessageFields))for _, field := range llmMessageFields {