Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(chat): preserve images across reasoning and conversation history (#20728)

Sbaaoui Idrissf6477b197768
diff --git a/conf/models/bedrock.json b/conf/models/bedrock.json
index b322175c44..b24e1aa7bd 100644
--- a/conf/models/bedrock.json
+++ b/conf/models/bedrock.json
@@ -12,7 +12,8 @@
"context_length": 200000,
"max_output": 8192,
"model_types": [
- "chat"
+ "chat",
+ "vision"
]
},
{
@@ -28,7 +29,8 @@
"context_length": 200000,
"max_output": 4096,
"model_types": [
- "chat"
+ "chat",
+ "vision"
]
},
{
@@ -36,7 +38,8 @@
"context_length": 200000,
"max_output": 4096,
"model_types": [
- "chat"
+ "chat",
+ "vision"
]
},
{
@@ -44,7 +47,8 @@
"context_length": 200000,
"max_output": 4096,
"model_types": [
- "chat"
+ "chat",
+ "vision"
]
},
{
@@ -92,7 +96,8 @@
"context_length": 300000,
"max_output": 5000,
"model_types": [
- "chat"
+ "chat",
+ "vision"
]
},
{
@@ -100,7 +105,8 @@
"context_length": 300000,
"max_output": 5000,
"model_types": [
- "chat"
+ "chat",
+ "vision"
]
},
{
diff --git a/conf/models/cohere.json b/conf/models/cohere.json
index e0c37bc01f..24fb86854c 100644
--- a/conf/models/cohere.json
+++ b/conf/models/cohere.json
@@ -18,7 +18,8 @@
"context_length": 131072,
"max_output": 65536,
"model_types": [
- "chat"
+ "chat",
+ "vision"
]
},
{
@@ -58,7 +59,8 @@
"context_length": 131072,
"max_output": 8192,
"model_types": [
- "chat"
+ "chat",
+ "vision"
]
},
{
diff --git a/conf/models/minimax.json b/conf/models/minimax.json
index cdd3ede5d3..59fd934680 100644
--- a/conf/models/minimax.json
+++ b/conf/models/minimax.json
@@ -18,7 +18,8 @@
"context_length": 1024000,
"max_output": 131072,
"model_types": [
- "chat"
+ "chat",
+ "vision"
],
"thinking": {
"default_value": true,
diff --git a/conf/models/mistral.json b/conf/models/mistral.json
index 11bd30eb16..2119046046 100644
--- a/conf/models/mistral.json
+++ b/conf/models/mistral.json
@@ -18,7 +18,8 @@
"context_length": 262144,
"max_output": 262144,
"model_types": [
- "chat"
+ "chat",
+ "vision"
],
"tools": {
"support": true
@@ -41,7 +42,8 @@
"context_length": 262144,
"max_output": 262144,
"model_types": [
- "chat"
+ "chat",
+ "vision"
],
"tools": {
"support": true
@@ -52,7 +54,8 @@
"context_length": 262144,
"max_output": 262144,
"model_types": [
- "chat"
+ "chat",
+ "vision"
],
"tools": {
"support": true
@@ -63,7 +66,8 @@
"context_length": 262144,
"max_output": 262144,
"model_types": [
- "chat"
+ "chat",
+ "vision"
],
"tools": {
"support": true
@@ -74,7 +78,8 @@
"context_length": 262144,
"max_output": 262144,
"model_types": [
- "chat"
+ "chat",
+ "vision"
],
"tools": {
"support": true
diff --git a/conf/models/novita.json b/conf/models/novita.json
index ae357c8784..234e01c3fa 100644
--- a/conf/models/novita.json
+++ b/conf/models/novita.json
@@ -72,7 +72,8 @@
"context_length": 98304,
"max_output": 16384,
"model_types": [
- "chat"
+ "chat",
+ "vision"
],
"tools": {
"support": true
diff --git a/conf/models/openai.json b/conf/models/openai.json
index 9f2cd70350..e0973fa050 100644
--- a/conf/models/openai.json
+++ b/conf/models/openai.json
@@ -357,7 +357,8 @@
"context_length": 128000,
"max_output": 4096,
"model_types": [
- "chat"
+ "chat",
+ "vision"
],
"tools": {
"support": true
diff --git a/conf/models/siliconflow.json b/conf/models/siliconflow.json
index 58caa25a81..87399b1a9e 100644
--- a/conf/models/siliconflow.json
+++ b/conf/models/siliconflow.json
@@ -917,7 +917,8 @@
"context_length": 262144,
"max_output": 262144,
"model_types": [
- "chat"
+ "chat",
+ "vision"
],
"tools": {
"support": true
diff --git a/internal/agentic_rag/agent.go b/internal/agentic_rag/agent.go
index 332393313a..86a38a9cca 100644
--- a/internal/agentic_rag/agent.go
+++ b/internal/agentic_rag/agent.go
@@ -390,9 +390,10 @@ func Run(ctx context.Context, in Input) (string, error) {
// EnableStreaming lives on RunnerConfig, not ChatModelAgentConfig.
explorerHead := sess.explorer.head(ctx)
- question := lastUserQuestion(in.Messages)
+ question := schema.UserMessage(lastUserQuestion(in.Messages))
seed := in.Messages
if len(seed) > 0 {
+ question = seed[len(seed)-1]
// The final user message is the new turn and must be appended exactly
// once by turnMessages.
seed = seed[:len(seed)-1]
@@ -721,10 +722,12 @@ func finalizeAnswer(
"assumption explicitly, and still provide the computed answer. Do not request any tools " +
"and do not hedge with \"I would need\" or \"I could not find\".")
- resp, err := m.Generate(ctx, []*schema.Message{{
+ finalMessages := append([]*schema.Message(nil), messages...)
+ finalMessages = append(finalMessages, &schema.Message{
Role: schema.User,
Content: body.String(),
- }})
+ })
+ resp, err := m.Generate(ctx, finalMessages)
if err != nil {
return "", err
}
diff --git a/internal/agentic_rag/run_session.go b/internal/agentic_rag/run_session.go
index a4fc1d8d0a..cff1d5c1f3 100644
--- a/internal/agentic_rag/run_session.go
+++ b/internal/agentic_rag/run_session.go
@@ -189,9 +189,9 @@ func (c *conversation) markSeeded() {
// deliverable first — a repair turn that lost its predecessor's events to a
// restart must still see the question it is answering — every later turn sends
// the directive alone and lets the Runner replay the rest.
-func (c *conversation) turnMessages(seed []*schema.Message, final, directive string) []adk.Message {
+func (c *conversation) turnMessages(seed []*schema.Message, final string, directive *schema.Message) []adk.Message {
if c.seeded {
- return []adk.Message{schema.UserMessage(directive)}
+ return []adk.Message{directive}
}
c.markSeeded()
msgs := make([]adk.Message, 0, len(seed)+2)
@@ -201,7 +201,7 @@ func (c *conversation) turnMessages(seed []*schema.Message, final, directive str
// standing deliverable the directive alone carries the ask.
msgs = append(msgs, schema.AssistantMessage(final, nil))
}
- return append(msgs, schema.UserMessage(directive))
+ return append(msgs, directive)
}
// lastAssistant walks the conversation's message events NEWEST FIRST and
diff --git a/internal/agentic_rag/run_session_test.go b/internal/agentic_rag/run_session_test.go
index 436ce16c20..5fe1c1479f 100644
--- a/internal/agentic_rag/run_session_test.go
+++ b/internal/agentic_rag/run_session_test.go
@@ -139,7 +139,7 @@ func TestConversationRestartWhenNothingCommitted(t *testing.T) {
t.Error("a restarted conversation must re-seed the caller's history")
}
base := []*schema.Message{schema.UserMessage("who signed?")}
- got := conv.turnMessages(base, "Final Answer: **1897**", "FIX IT")
+ got := conv.turnMessages(base, "Final Answer: **1897**", schema.UserMessage("FIX IT"))
if len(got) != 3 || got[0].Content != "who signed?" || got[2].Content != "FIX IT" {
t.Fatalf("reseeded turn = %v, want history + final + directive", got)
}
@@ -153,15 +153,29 @@ func TestConversationRestartWhenNothingCommitted(t *testing.T) {
// The caller's chat history belongs to ONE turn: later turns hand over nothing
// but their own message.
+func TestConversationTurnMessagesPreservesImages(t *testing.T) {
+ conv := newTestConversation()
+ uri := "data:image/jpeg;base64,AAAA"
+ question := schema.UserMessage("describe")
+ question.UserInputMultiContent = []schema.MessageInputPart{{
+ Type: schema.ChatMessagePartTypeImageURL,
+ Image: &schema.MessageInputImage{MessagePartCommon: schema.MessagePartCommon{URL: &uri}},
+ }}
+ got := conv.turnMessages(nil, "", question)
+ if len(got) != 1 || len(got[0].UserInputMultiContent) != 1 {
+ t.Fatalf("got %#v, want image-bearing user turn", got)
+ }
+}
+
func TestConversationTurnMessagesSeedsOnce(t *testing.T) {
conv := newTestConversation()
base := []*schema.Message{schema.UserMessage("who signed?")}
- first := conv.turnMessages(base, "", "FIX IT")
+ first := conv.turnMessages(base, "", schema.UserMessage("FIX IT"))
if len(first) != 2 || first[0].Content != "who signed?" || first[1].Content != "FIX IT" {
t.Fatalf("unseeded turn = %v, want history + directive", first)
}
- second := conv.turnMessages(base, "Final Answer: **1897**", "AGAIN")
+ second := conv.turnMessages(base, "Final Answer: **1897**", schema.UserMessage("AGAIN"))
if len(second) != 1 || second[0].Content != "AGAIN" {
t.Fatalf("seeded turn = %v, want the directive alone", second)
}
diff --git a/internal/service/agentic_pipeline.go b/internal/service/agentic_pipeline.go
index 5507e8b961..4ae0c4394d 100644
--- a/internal/service/agentic_pipeline.go
+++ b/internal/service/agentic_pipeline.go
@@ -114,6 +114,24 @@ func (s *ChatPipelineService) agenticRag(
return
}
docIDs := s.chatMetadataDocIDs(runCtx, chat, lastUserQuestion(messages), chain[0], nil, chatDatasetIDs(chat))
+ modelType := "chat"
+ if info := chain[0].Info(); info != nil && entity.ModelTypeFromStrings(info.ModelTypes).Has(entity.ModelTypeImage2Text) {
+ modelType = "image2text"
+ }
+ agentMessages := s.convertMessagesToEino(runCtx, userID, modelType, messages)
+ // Image-bearing turns cannot fail over to a text-only model.
+ for _, msg := range agentMessages {
+ if len(msg.UserInputMultiContent) > 0 {
+ visionChain := chain[:0]
+ for _, candidate := range chain {
+ if info := candidate.Info(); info != nil && entity.ModelTypeFromStrings(info.ModelTypes).Has(entity.ModelTypeImage2Text) {
+ visionChain = append(visionChain, candidate)
+ }
+ }
+ chain = visionChain
+ break
+ }
+ }
// Apply the dialog's LLM settings with per-request overrides exactly
// like the regular AsyncChat path does, then let the template pin its
@@ -163,7 +181,7 @@ func (s *ChatPipelineService) agenticRag(
final, runErr := agentic_rag.Run(runCtx, agentic_rag.Input{
Model: model,
SynthModel: synth,
- Messages: convertMessagesToEino(messages),
+ Messages: agentMessages,
TemplateID: mode,
TenantID: chat.TenantID,
DatasetIDs: chatDatasetIDs(chat),
@@ -352,16 +370,30 @@ func (s *ChatPipelineService) agenticWebSearch(promptConfig map[string]interface
}
// convertMessagesToEino converts pre-filtered user/assistant messages into
-// eino schema messages. Only string content is supported; multimodal parts are
-// not carried into the ReAct loop.
-func convertMessagesToEino(messages []map[string]interface{}) []*schema.Message {
+// eino schema messages, resolving uploaded attachments throughout the history.
+func (s *ChatPipelineService) convertMessagesToEino(ctx context.Context, userID, modelType string, messages []map[string]interface{}) []*schema.Message {
out := make([]*schema.Message, 0, len(messages))
for _, m := range messages {
role, _ := m["role"].(string)
content, _ := m["content"].(string)
switch role {
case "user":
- out = append(out, schema.UserMessage(content))
+ text, images := s.splitChatAttachments(ctx, userID, m["files"])
+ if text != "" {
+ content += "\n\n" + text
+ }
+ msg := schema.UserMessage(content)
+ images, _ = gateImageAttachments("", modelType, images)
+ if len(images) > 0 {
+ msg.UserInputMultiContent = append(msg.UserInputMultiContent, schema.MessageInputPart{Type: schema.ChatMessagePartTypeText, Text: content})
+ for _, uri := range images {
+ msg.UserInputMultiContent = append(msg.UserInputMultiContent, schema.MessageInputPart{
+ Type: schema.ChatMessagePartTypeImageURL,
+ Image: &schema.MessageInputImage{MessagePartCommon: schema.MessagePartCommon{URL: new(uri)}},
+ })
+ }
+ }
+ out = append(out, msg)
case "assistant":
out = append(out, schema.AssistantMessage(content, nil))
default:
diff --git a/internal/service/chat_pipeline.go b/internal/service/chat_pipeline.go
index 5b2c9518fd..99360b4436 100644
--- a/internal/service/chat_pipeline.go
+++ b/internal/service/chat_pipeline.go
@@ -438,6 +438,17 @@ func (s *ChatPipelineService) AsyncChat(
zap.Int("attachment_bytes", len(attachments)))
}
+ var historyImageFiles []string
+ for _, previous := range messages[:len(messages)-1] {
+ if previous["role"] == "user" && previous["files"] != nil {
+ _, images := s.splitChatAttachments(ctx, userID, previous["files"])
+ modelType, _ := llmModelConfig["model_type"].(string)
+ images, _ = gateImageAttachments(chat.LLMID, modelType, images)
+ historyImageFiles = append(historyImageFiles, images...)
+ }
+ }
+ hasImageAttachments = hasImageAttachments || len(historyImageFiles) > 0
+
// === Phase 6: SQL Retrieval ===
// Retrieve field_map for SQL retrieval (preferred over vector search)
promptConfig := chat.PromptConfig
@@ -835,11 +846,12 @@ func (s *ChatPipelineService) AsyncChat(
}
harnessSystemPrompt = s.formatPrompt(sp, kws)
}
+ harnessImages := append(historyImageFiles, imageFiles...)
var history []map[string]interface{}
if kwargs["store_history_messages"] == false {
history = messages
}
- hk, slotCites, citeChunkIDs, harnessAnswer, hErr := s.retrieveViaHarness(ctx, question, kbs, docIDs, imageFiles, attachments, thinkingMode, chat.TenantID, chat.LLMID, chat.ID, HarnessRetrieval{
+ hk, slotCites, citeChunkIDs, harnessAnswer, hErr := s.retrieveViaHarness(ctx, question, kbs, docIDs, harnessImages, attachments, thinkingMode, chat.TenantID, chat.LLMID, chat.ID, HarnessRetrieval{
TopN: int(chat.TopN),
SimilarityThreshold: chat.SimilarityThreshold,
VectorSimilarityWeight: chat.VectorSimilarityWeight,
@@ -1134,12 +1146,16 @@ func (s *ChatPipelineService) AsyncChat(
if factoryName == "" {
factoryName = factoryFromLLMID(chat.LLMID)
}
- for _, m := range messages {
+ modelType, _ := llmModelConfig["model_type"].(string)
+ for i, m := range messages {
role, _ := m["role"].(string)
if role == "system" && kwargs["store_history_messages"] != false {
continue
}
llmMessage := normalizeLLMMessage(m)
+ if i < len(messages)-1 {
+ llmMessage["files"] = m["files"]
+ }
content := llmMessage["content"]
if contentStr, ok := content.(string); ok {
content = cleanCitationMarkers(contentStr)
@@ -1155,6 +1171,9 @@ func (s *ChatPipelineService) AsyncChat(
// Fit messages within token budget.
usedTokenCount, llmMessages := s.messageFitIn(llmMessages, int(float64(modelMaxTokens)*0.95))
+ for i := range llmMessages[:len(llmMessages)-1] {
+ llmMessages[i] = s.chatHistoryMessage(ctx, userID, llmMessages[i], modelType, factoryName)
+ }
common.Debug("Messages fitted in token budget",
zap.Int("model max_tokens", modelMaxTokens),
zap.Int("used_token_count", usedTokenCount),
@@ -1625,11 +1644,14 @@ func (s *ChatPipelineService) AsyncChatSolo(
// 3. Strip citation markers; non-storing model tests retain payload system messages.
var msg []map[string]interface{}
- for _, m := range messages {
+ for i, m := range messages {
role, _ := m["role"].(string)
if role == "system" && config["store_history_messages"] != false {
continue
}
+ if i < len(messages)-1 {
+ m = s.chatHistoryMessage(ctx, userID, m, modelType, factoryName)
+ }
llmMessage := normalizeLLMMessage(m)
content := llmMessage["content"]
if contentStr, ok := content.(string); ok {
@@ -2854,6 +2876,21 @@ func (s *ChatPipelineService) buildChatMessages(systemContent string, messages [
return result
}
+func (s *ChatPipelineService) chatHistoryMessage(ctx context.Context, userID string, message map[string]interface{}, modelType, factory string) map[string]interface{} {
+ if message["role"] != "user" || message["files"] == nil {
+ return message
+ }
+ _, images := s.splitChatAttachments(ctx, userID, message["files"])
+ images, _ = gateImageAttachments("", modelType, images)
+ msg := normalizeLLMMessage(message)
+ converted, err := common.ConvertLastUserMsgToMultimodal(msg, images, factory)
+ if err != nil {
+ common.Warn("chat history multimodal conversion failed", zap.Error(err))
+ return msg
+ }
+ return converted
+}
+
func normalizeLLMMessage(message map[string]interface{}) map[string]interface{} {
normalized := make(map[string]interface{}, len(llmMessageFields))
for _, field := range llmMessageFields {