Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

refactor(agent): use shared message-fit in LLM component (#18092)

jay77721341e913b7f9c
diff --git a/internal/agent/component/llm.go b/internal/agent/component/llm.go
index 99720ce9c8..a7387814e2 100644
--- a/internal/agent/component/llm.go
+++ b/internal/agent/component/llm.go
@@ -27,8 +27,8 @@ import (
"ragflow/internal/agent/component/prompts"
"ragflow/internal/agent/runtime"
"ragflow/internal/common"
+ "ragflow/internal/component/messagefit"
"ragflow/internal/entity/models"
- "ragflow/internal/tokenizer"
"go.uber.org/zap"
)
@@ -309,7 +309,6 @@ func (c *LLMComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[strin
if err != nil {
return nil, fmt.Errorf("component: LLM.Invoke: resolve model: %w", err)
}
-
if p.ModelID == "" {
return nil, &ParamError{Field: "model_id", Reason: "required"}
}
@@ -1024,8 +1023,6 @@ func validateFittedMessages(msgFit []schema.Message) string {
// fitted messages and an error string (empty on success).
// Mirrors Python's LLM.fit_messages in PR #16413.
func fitMessages(systemPrompt string, msgs []schema.Message, maxLength int) ([]schema.Message, string) {
- // Convert schema.Message → []map[string]interface{} for fitting.
- all := make([]map[string]interface{}, 0, 1+len(msgs))
// Deep-copy msgs (mirrors Python's deepcopy) to avoid mutating caller's slice.
copied := make([]schema.Message, len(msgs))
for i, m := range msgs {
@@ -1046,162 +1043,87 @@ func fitMessages(systemPrompt string, msgs []schema.Message, maxLength int) ([]s
UserInputMultiContent: cloned,
}
}
- // System prompt first (when not already embedded in msgs).
- if systemPrompt != "" {
- all = append(all, map[string]interface{}{
- "role": "system",
- "content": systemPrompt,
- })
- }
- for _, m := range copied {
- entry := map[string]interface{}{
- "role": string(m.Role),
- "content": m.Content,
- }
- if len(m.UserInputMultiContent) > 0 {
- entry["user_input_multi_content"] = m.UserInputMultiContent
- }
- all = append(all, entry)
- }
- // Use 97% of effective context as the token budget.
- budget := contextFitBudget(maxLength)
- _, fitted := messageFitInRaw(all, budget)
-
- // Convert back to []schema.Message.
- result := make([]schema.Message, 0, len(fitted))
- for _, m := range fitted {
- role, _ := m["role"].(string)
- content, _ := m["content"].(string)
- msg := schema.Message{
- Role: schema.RoleType(role),
- Content: content,
- }
- if multi, ok := m["user_input_multi_content"].([]schema.MessageInputPart); ok {
- msg.UserInputMultiContent = multi
- }
- result = append(result, msg)
- }
- return result, validateFittedMessages(result)
-}
-
-// messageFitInRaw trims messages to fit within a token budget. Operates on
-// raw []map[string]interface{} (role + content). Returns the token count
-// used and the trimmed slice. Mirrors Python's message_fit_in in
-// rag/prompts/generator.py.
-//
-// Strategy:
-// 1. If everything fits → return as-is.
-// 2. Keep all system messages + the last user/assistant message.
-// 3. If still too large, trim content proportionally:
-// - System dominates (>80%) → preserve last message first.
-// - Otherwise → preserve system first.
-func messageFitInRaw(messages []map[string]interface{}, maxTokens int) (int, []map[string]interface{}) {
- if maxTokens <= 0 {
- maxTokens = 8192
- }
-
- // Step 1: everything fits.
- totalTokens := countAllTokens(messages)
- if totalTokens < maxTokens {
- return totalTokens, messages
- }
-
- // Step 2: keep all system messages + the last non-system message.
- result := make([]map[string]interface{}, 0)
- for _, m := range messages {
- if role, _ := m["role"].(string); role == "system" {
- result = append(result, m)
- }
- }
- if len(messages) > 0 {
- last := messages[len(messages)-1]
- if role, _ := last["role"].(string); role != "system" {
- result = append(result, last)
- }
- }
- if len(result) == 0 {
- return 0, result
- }
-
- totalTokens = countAllTokens(result)
- if totalTokens < maxTokens {
- return totalTokens, result
- }
- // Step 3: trim content to fit.
- ll := tokenizer.NumTokensFromString(stringContent(result[0]))
- ll2 := tokenizer.NumTokensFromString(stringContent(result[len(result)-1]))
- total := ll + ll2
- if total <= 0 {
- return 0, result
+ // Convert to messagefit.Message. Track where each entry's text lives
+ // (plain Content or a multi-modal text part) so the fitted text can be
+ // written back to the right field. Entries with no text at all
+ // (image-only turns) carry an empty Content in messagefit and survive
+ // fitting when kept.
+ type fitSource struct {
+ copiedIdx int // index into copied; -1 for the synthetic system prompt
+ multiIdx int // -1 means the text lives in Content
+ textInContent bool // the original message carried text in Content
}
+ all := make([]messagefit.Message, 0, 1+len(copied))
+ sources := make([]fitSource, 0, 1+len(copied))
- if len(result) == 1 {
- setContent(result[0], tokenizer.TrimContentToTokenLimit(stringContent(result[0]), maxTokens))
- return countAllTokens(result), result
- }
-
- if float64(ll)/float64(total) > 0.8 {
- preservedLast := min(ll2, maxTokens)
- setContent(result[len(result)-1], tokenizer.TrimContentToTokenLimit(stringContent(result[len(result)-1]), preservedLast))
- remaining := max(0, maxTokens-preservedLast)
- setContent(result[0], tokenizer.TrimContentToTokenLimit(stringContent(result[0]), remaining))
- } else {
- preservedSystem := min(ll, maxTokens)
- setContent(result[0], tokenizer.TrimContentToTokenLimit(stringContent(result[0]), preservedSystem))
- remaining := max(0, maxTokens-preservedSystem)
- setContent(result[len(result)-1], tokenizer.TrimContentToTokenLimit(stringContent(result[len(result)-1]), remaining))
- }
-
- return countAllTokens(result), result
-}
-
-// countAllTokens returns the total token count across all messages.
-func countAllTokens(messages []map[string]interface{}) int {
- total := 0
- for _, m := range messages {
- total += tokenizer.NumTokensFromString(stringContent(m))
- }
- return total
-}
-
-// stringContent extracts the display text from a message map, or "".
-// For plain messages the text lives in "content"; for multimodal messages
-// (with images) it lives in a text part of "user_input_multi_content".
-func stringContent(m map[string]interface{}) string {
- s, _ := m["content"].(string)
- if s != "" {
- return s
- }
- if multi, ok := m["user_input_multi_content"].([]schema.MessageInputPart); ok {
- for _, part := range multi {
- if part.Type == schema.ChatMessagePartTypeText && part.Text != "" {
- return part.Text
+ if systemPrompt != "" {
+ all = append(all, messagefit.Message{Role: "system", Content: systemPrompt})
+ sources = append(sources, fitSource{copiedIdx: -1, multiIdx: 0})
+ }
+
+ for i := range copied {
+ text := copied[i].Content
+ multiIdx := -1
+ hadText := text != ""
+ if !hadText {
+ // Fold every non-empty text part into the token budget: only the
+ // first text part is written back, so leaving later parts out
+ // would let text exceed the fitted budget after reconstruction.
+ var textParts []string
+ for j, p := range copied[i].UserInputMultiContent {
+ if p.Type == schema.ChatMessagePartTypeText && p.Text != "" {
+ textParts = append(textParts, p.Text)
+ if multiIdx < 0 {
+ multiIdx = j
+ }
+ }
+ }
+ if len(textParts) > 0 {
+ text = strings.Join(textParts, "\n\n")
+ hadText = true
}
}
+ all = append(all, messagefit.Message{Role: string(copied[i].Role), Content: text})
+ sources = append(sources, fitSource{copiedIdx: i, multiIdx: multiIdx, textInContent: copied[i].Content != ""})
}
- return ""
-}
-// setContent writes text into a message map's display field. When the
-// message has user_input_multi_content parts, the first text part is
-// updated; otherwise the plain "content" field is set.
-func setContent(m map[string]interface{}, text string) {
- if multi, ok := m["user_input_multi_content"].([]schema.MessageInputPart); ok {
- for i, part := range multi {
- if part.Type == schema.ChatMessagePartTypeText {
- multi[i].Text = text
- return
+ // Use 97% of effective context as the token budget.
+ budget := contextFitBudget(maxLength)
+ kept, keptIdx, _ := messagefit.Fit(all, budget)
+
+ // Convert back to []schema.Message. messagefit.Fit reports exactly which
+ // entries are kept (keptIdx); dropped entries are simply absent, so no
+ // empty-content sentinel is needed and image-only turns are preserved.
+ result := make([]schema.Message, 0, len(kept))
+ for j, i := range keptIdx {
+ src := sources[i]
+ if src.copiedIdx < 0 {
+ result = append(result, schema.Message{Role: schema.System, Content: kept[j].Content})
+ continue
+ }
+ m := copied[src.copiedIdx]
+ if src.multiIdx >= 0 && src.multiIdx < len(m.UserInputMultiContent) {
+ m.UserInputMultiContent[src.multiIdx].Text = kept[j].Content
+ // Drop any additional text parts: their content was folded into
+ // the first part before fitting, so keeping them would re-introduce
+ // text outside the token budget.
+ keptParts := m.UserInputMultiContent[:0]
+ for k, part := range m.UserInputMultiContent {
+ if part.Type == schema.ChatMessagePartTypeText && k != src.multiIdx {
+ continue
+ }
+ keptParts = append(keptParts, part)
}
+ m.UserInputMultiContent = keptParts
+ } else if src.textInContent {
+ // Always write the fitted text back (even when trimmed to empty):
+ // leaving the original would send untrimmed content past the budget.
+ m.Content = kept[j].Content
}
- // No existing text part – prepend one.
- m["user_input_multi_content"] = append(
- []schema.MessageInputPart{{Type: schema.ChatMessagePartTypeText, Text: text}},
- multi...,
- )
- return
+ result = append(result, m)
}
- m["content"] = text
+ return result, validateFittedMessages(result)
}
// stringFrom extracts a string from inputs[name], accepting both string and
diff --git a/internal/agent/component/llm_test.go b/internal/agent/component/llm_test.go
index 9f0625a949..d56e1b794b 100644
--- a/internal/agent/component/llm_test.go
+++ b/internal/agent/component/llm_test.go
@@ -16,9 +16,11 @@ import (
"context"
"errors"
"slices"
+ "strings"
"testing"
"ragflow/internal/entity"
+ "ragflow/internal/tokenizer"
"github.com/cloudwego/eino/schema"
"gorm.io/gorm"
@@ -319,3 +321,148 @@ func TestLLM_ResolvesTenantModelID(t *testing.T) {
t.Errorf("BaseURL=%q, want %q", got, want)
}
}
+
+func TestFitMessages_EverythingFits(t *testing.T) {
+ msgs := []schema.Message{
+ {Role: schema.System, Content: "you are helpful"},
+ {Role: schema.User, Content: "hello"},
+ }
+ fitted, fitErr := fitMessages("", msgs, 100000)
+ if fitErr != "" {
+ t.Fatalf("unexpected fit error: %s", fitErr)
+ }
+ if len(fitted) != 2 {
+ t.Fatalf("got %d messages, want 2", len(fitted))
+ }
+ if fitted[0].Content != "you are helpful" || fitted[1].Content != "hello" {
+ t.Fatalf("messages modified when they fit: %+v", fitted)
+ }
+}
+
+func TestFitMessages_PreservesImageOnlyTurn(t *testing.T) {
+ imgURL := "data:image/png;base64,AAAA"
+ msgs := []schema.Message{
+ {Role: schema.System, Content: "you are helpful"},
+ {Role: schema.User, UserInputMultiContent: []schema.MessageInputPart{
+ {Type: schema.ChatMessagePartTypeImageURL, Image: &schema.MessageInputImage{
+ MessagePartCommon: schema.MessagePartCommon{URL: &imgURL},
+ }},
+ }},
+ }
+ fitted, fitErr := fitMessages("", msgs, 100000)
+ if fitErr != "" {
+ t.Fatalf("unexpected fit error: %s", fitErr)
+ }
+ if len(fitted) != 2 {
+ t.Fatalf("got %d messages, want 2 (image-only turn must be preserved)", len(fitted))
+ }
+ last := fitted[len(fitted)-1]
+ if len(last.UserInputMultiContent) != 1 || last.UserInputMultiContent[0].Type != schema.ChatMessagePartTypeImageURL {
+ t.Fatalf("image parts lost after fitting: %+v", last)
+ }
+}
+
+func TestFitMessages_IncludesSyntheticSystemPrompt(t *testing.T) {
+ msgs := []schema.Message{{Role: schema.User, Content: "hello"}}
+ fitted, fitErr := fitMessages("be brief", msgs, 100000)
+ if fitErr != "" {
+ t.Fatalf("unexpected fit error: %s", fitErr)
+ }
+ if len(fitted) != 2 {
+ t.Fatalf("got %d messages, want 2 (synthetic system prompt + user)", len(fitted))
+ }
+ if fitted[0].Role != schema.System || fitted[0].Content != "be brief" {
+ t.Fatalf("synthetic system prompt not preserved: %+v", fitted[0])
+ }
+}
+
+func TestFitMessages_DropsMiddleWhenOverBudget(t *testing.T) {
+ long := strings.Repeat("x ", 5000)
+ msgs := []schema.Message{
+ {Role: schema.System, Content: long},
+ {Role: schema.User, Content: "middle"},
+ {Role: schema.User, Content: "last"},
+ }
+ fitted, fitErr := fitMessages("", msgs, 1000)
+ if fitErr != "" {
+ t.Fatalf("unexpected fit error: %s", fitErr)
+ }
+ if len(fitted) != 2 {
+ t.Fatalf("got %d messages, want 2 (middle dropped, system + last user kept)", len(fitted))
+ }
+ if fitted[0].Role != schema.System || fitted[1].Role != schema.User {
+ t.Fatalf("unexpected roles: %+v", fitted)
+ }
+ if !strings.Contains(fitted[1].Content, "last") {
+ t.Fatalf("last user message not preserved: %+v", fitted[1])
+ }
+}
+
+// TestFitMessages_SystemKeptButEmptied locks the write-back for a system
+// message that the fitter keeps but trims to empty (the final user turn alone
+// fills the budget): the fitted (empty) content must be written back instead
+// of the original, so the conversation stays within the budget.
+func TestFitMessages_SystemKeptButEmptied(t *testing.T) {
+ origSys := strings.Repeat("s ", 3000) // dominates (>80% of tokens)
+ msgs := []schema.Message{
+ {Role: schema.System, Content: origSys},
+ {Role: schema.User, Content: strings.Repeat("u ", 600)}, // alone exceeds the budget
+ }
+ fitted, fitErr := fitMessages("", msgs, 500)
+ if fitErr != "" {
+ t.Fatalf("unexpected fit error: %s", fitErr)
+ }
+ if len(fitted) != 2 {
+ t.Fatalf("got %d messages, want 2 (both kept)", len(fitted))
+ }
+ if fitted[0].Role != schema.System || fitted[0].Content != "" {
+ t.Fatalf("system should be kept but trimmed to empty, got %+v", fitted[0])
+ }
+ if fitted[1].Role != schema.User || fitted[1].Content == origSys {
+ t.Fatalf("user turn wrong after fitting: %+v", fitted[1])
+ }
+ total := tokenizer.NumTokensFromString(fitted[0].Content) + tokenizer.NumTokensFromString(fitted[1].Content)
+ if total > 500 {
+ t.Fatalf("fitted total %d exceeds budget 500", total)
+ }
+}
+
+// TestFitMessages_FoldsMultipleTextParts verifies that every non-empty text
+// part of a multi-modal message participates in the token budget: the parts
+// are folded into a single fitted text on the first text part and additional
+// text parts are removed, so no text escapes the budget after reconstruction.
+func TestFitMessages_FoldsMultipleTextParts(t *testing.T) {
+ long1 := strings.Repeat("a ", 3000)
+ long2 := strings.Repeat("b ", 3000)
+ imgURL := "data:image/png;base64,AAAA"
+ msgs := []schema.Message{
+ {Role: schema.System, Content: "sys"},
+ {Role: schema.User, UserInputMultiContent: []schema.MessageInputPart{
+ {Type: schema.ChatMessagePartTypeText, Text: long1},
+ {Type: schema.ChatMessagePartTypeImageURL, Image: &schema.MessageInputImage{
+ MessagePartCommon: schema.MessagePartCommon{URL: &imgURL},
+ }},
+ {Type: schema.ChatMessagePartTypeText, Text: long2},
+ }},
+ }
+ fitted, fitErr := fitMessages("", msgs, 2000)
+ if fitErr != "" {
+ t.Fatalf("unexpected fit error: %s", fitErr)
+ }
+ if len(fitted) != 2 {
+ t.Fatalf("got %d messages, want 2", len(fitted))
+ }
+ last := fitted[len(fitted)-1]
+ textParts := 0
+ for _, part := range last.UserInputMultiContent {
+ if part.Type == schema.ChatMessagePartTypeText {
+ textParts++
+ }
+ }
+ if textParts != 1 {
+ t.Fatalf("got %d text parts, want 1 (folded): %+v", textParts, last.UserInputMultiContent)
+ }
+ if total := tokenizer.NumTokensFromString(last.UserInputMultiContent[0].Text); total > 2000 {
+ t.Fatalf("fitted text totals %d tokens, exceeds budget 2000", total)
+ }
+}